Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,8 @@ Because embedded gem dependency sometimes restricts ruby environment.
| :------------------------------------- | :------------ | :----------- | :---------- | :------------------------- | :----------------------- |
| source_format | enum | no | no | json | Specify source format `json` or `csv` or `avro`. If you change this parameter, you must change formatter plugin via `<format>` config section. |
| max_bad_records | integer | no | no | 0 | If the number of bad records exceeds this value, an invalid error is returned in the job result. |
| compression | enum | no | no | none | Compress the data uploaded for a load job. `none` or `gzip`. `gzip` reduces network egress, but load jobs may take longer because BigQuery cannot read gzip-compressed files in parallel. Not supported with `source_format avro`. |
| compression_level | integer | no | no | -1 | zlib compression level (0-9) used when `compression gzip`. `-1` uses zlib's default (equivalent to 6). |

### Buffer section

Expand Down
46 changes: 45 additions & 1 deletion lib/fluent/plugin/out_bigquery_load.rb
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
require 'zlib'
require 'fluent/plugin/out_bigquery_base'

module Fluent
Expand All @@ -9,6 +10,13 @@ class BigQueryLoadOutput < BigQueryBaseOutput

config_param :source_format, :enum, list: [:json, :avro, :csv], default: :json

# compression (only load)
# Compress the data uploaded for a load job. BigQuery detects gzip-compressed input automatically.
# This reduces the bytes sent over the network at the cost of CPU on the fluentd side and
# slower load jobs on the BigQuery side, because BigQuery cannot read a gzip file in parallel.
config_param :compression, :enum, list: [:none, :gzip], default: :none
config_param :compression_level, :integer, default: Zlib::DEFAULT_COMPRESSION

# max_bad_records (only load)
# The maximum number of bad records that BigQuery can ignore when running the job.
# If the number of bad records exceeds this value, an invalid error is returned in the job result.
Expand Down Expand Up @@ -38,6 +46,15 @@ def configure(conf)
super
@is_load = true

if @compression == :gzip
unless @compression_level == Zlib::DEFAULT_COMPRESSION || (Zlib::NO_COMPRESSION..Zlib::BEST_COMPRESSION).cover?(@compression_level)
raise Fluent::ConfigError, "compression_level must be between #{Zlib::NO_COMPRESSION} and #{Zlib::BEST_COMPRESSION}, or #{Zlib::DEFAULT_COMPRESSION} for zlib's default"
end
if @source_format == :avro
raise Fluent::ConfigError, "compression gzip is not supported with source_format avro"
end
end

placeholder_params = "project=#{@project}/dataset=#{@dataset}/table=#{@tablelist.join(",")}/fetch_schema_table=#{@fetch_schema_table}"
placeholder_validate!(:bigquery_load, placeholder_params)
end
Expand Down Expand Up @@ -201,7 +218,11 @@ def poll
end
end

def create_upload_source(chunk)
def create_upload_source(chunk, &block)
if @compression == :gzip
return create_gzip_upload_source(chunk, &block)
end

chunk_is_file = @buffer_config["@type"] == 'file'
if chunk_is_file
File.open(chunk.path) do |file|
Expand All @@ -217,6 +238,29 @@ def create_upload_source(chunk)
end
end
end

def create_gzip_upload_source(chunk)
Tempfile.create("chunk-gzip-tmp") do |file|
file.binmode
started_at = Fluent::Clock.now
gz = Zlib::GzipWriter.new(file, @compression_level)
begin
chunk.write_to(gz)
raw_size = gz.pos
ensure
gz.finish
end
file.flush
compressed_size = file.size
log.debug("compressed chunk for load job",
chunk: dump_unique_id_hex(chunk.unique_id),
raw_bytes: raw_size,
compressed_bytes: compressed_size,
elapsed: (Fluent::Clock.now - started_at).round(3))
file.rewind
yield file
end
end
end
end
end
69 changes: 69 additions & 0 deletions test/plugin/test_out_bigquery_load.rb
Original file line number Diff line number Diff line change
Expand Up @@ -391,6 +391,75 @@ def test_write_with_auto_create_table_time_partitioning_expiration_in_seconds
assert_equal({"type" => "DAY", "expirationMs" => "2592000000"}, JSON.parse(json)["configuration"]["load"]["timePartitioning"])
end

def test_configure_compression_default
driver = create_driver
assert_equal :none, driver.instance.compression
end

def test_configure_compression_invalid_level
assert_raise(Fluent::ConfigError) do
create_driver(CONFIG + "compression gzip\ncompression_level 10\n")
end
end

def test_configure_compression_gzip_with_avro
assert_raise(Fluent::ConfigError) do
create_driver(CONFIG + "compression gzip\nsource_format avro\n")
end
end

data(
"memory buffer" => "",
"memory buffer with compress gzip" => "compress gzip",
)
def test_write_with_compression_gzip(buffer_option)
driver = create_driver(<<-CONFIG)
table foo
email foo@bar.example
private_key_path /path/to/key
project yourproject_id
dataset yourdataset_id

<buffer>
@type memory
#{buffer_option}
</buffer>

schema_path #{SCHEMA_PATH}
wait_job_interval 0.1
compression gzip
CONFIG

uploaded = nil
response_stub = stub!
stub_writer do |writer|
mock(writer).fetch_load_job(is_a(Fluent::BigQuery::Writer::JobReference)) { response_stub }
mock(writer).commit_load_job(is_a(String), response_stub)

mock(writer.client).get_table('yourproject_id', 'yourdataset_id', 'foo') { nil }

mock(writer.client).insert_job('yourproject_id', anything, upload_source: duck_type(:read, :rewind), content_type: "application/octet-stream") do |_, _, options|
uploaded = options[:upload_source].read
stub!.job_reference.stub! do |s|
s.job_id { "dummy_job_id" }
s.location { "us" }
end
end
end

driver.run do
driver.feed("tag", Time.now.to_i, {"a" => "b"})
driver.feed("tag", Time.now.to_i, {"a" => "c"})
end

# The upload source must be a single gzip stream of newline delimited JSON
assert_equal "\x1f\x8b".b, uploaded.byteslice(0, 2)
reader = Zlib::GzipReader.new(StringIO.new(uploaded))
lines = reader.read.each_line.map { |line| JSON.parse(line) }
assert_true reader.unused.nil?
assert_equal [{"a" => "b"}, {"a" => "c"}], lines
end

private

def create_response_stub(response)
Expand Down