From 7ad82e7f1830d081d3cd6351396d015669c611b6 Mon Sep 17 00:00:00 2001 From: keito-osaki Date: Mon, 16 Nov 2026 16:48:24 +0900 Subject: [PATCH] Add compression option to bigquery_load output Add `compression` (`none` / `gzip`) and `compression_level` parameters to the bigquery_load output. With `compression gzip`, each chunk is gzip-compressed into a temporary file before it is uploaded for the load job. BigQuery detects gzip-compressed input automatically, so no change to the job configuration is needed. This reduces network egress for newline-delimited JSON / CSV uploads, which typically compress to 10-30% of their size. The trade-off is CPU on the fluentd side and slower load jobs for large chunks, because BigQuery cannot read a gzip file in parallel. `source_format avro` is rejected because BigQuery does not accept gzip-compressed Avro files. The chunk is written through `chunk.write_to`, so a buffer configured with `compress gzip` is decompressed first and re-compressed into a single gzip stream. Co-Authored-By: Claude Opus 5.5 --- README.md | 2 + lib/fluent/plugin/out_bigquery_load.rb | 46 ++++++++++++++++- test/plugin/test_out_bigquery_load.rb | 69 ++++++++++++++++++++++++++ 3 files changed, 116 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 190a884..741804a 100644 --- a/README.md +++ b/README.md @@ -80,6 +80,8 @@ Because embedded gem dependency sometimes restricts ruby environment. | :------------------------------------- | :------------ | :----------- | :---------- | :------------------------- | :----------------------- | | source_format | enum | no | no | json | Specify source format `json` or `csv` or `avro`. If you change this parameter, you must change formatter plugin via `` config section. | | max_bad_records | integer | no | no | 0 | If the number of bad records exceeds this value, an invalid error is returned in the job result. | +| compression | enum | no | no | none | Compress the data uploaded for a load job. `none` or `gzip`. `gzip` reduces network egress, but load jobs may take longer because BigQuery cannot read gzip-compressed files in parallel. Not supported with `source_format avro`. | +| compression_level | integer | no | no | -1 | zlib compression level (0-9) used when `compression gzip`. `-1` uses zlib's default (equivalent to 6). | ### Buffer section diff --git a/lib/fluent/plugin/out_bigquery_load.rb b/lib/fluent/plugin/out_bigquery_load.rb index b2e8081..2c68019 100644 --- a/lib/fluent/plugin/out_bigquery_load.rb +++ b/lib/fluent/plugin/out_bigquery_load.rb @@ -1,3 +1,4 @@ +require 'zlib' require 'fluent/plugin/out_bigquery_base' module Fluent @@ -9,6 +10,13 @@ class BigQueryLoadOutput < BigQueryBaseOutput config_param :source_format, :enum, list: [:json, :avro, :csv], default: :json + # compression (only load) + # Compress the data uploaded for a load job. BigQuery detects gzip-compressed input automatically. + # This reduces the bytes sent over the network at the cost of CPU on the fluentd side and + # slower load jobs on the BigQuery side, because BigQuery cannot read a gzip file in parallel. + config_param :compression, :enum, list: [:none, :gzip], default: :none + config_param :compression_level, :integer, default: Zlib::DEFAULT_COMPRESSION + # max_bad_records (only load) # The maximum number of bad records that BigQuery can ignore when running the job. # If the number of bad records exceeds this value, an invalid error is returned in the job result. @@ -38,6 +46,15 @@ def configure(conf) super @is_load = true + if @compression == :gzip + unless @compression_level == Zlib::DEFAULT_COMPRESSION || (Zlib::NO_COMPRESSION..Zlib::BEST_COMPRESSION).cover?(@compression_level) + raise Fluent::ConfigError, "compression_level must be between #{Zlib::NO_COMPRESSION} and #{Zlib::BEST_COMPRESSION}, or #{Zlib::DEFAULT_COMPRESSION} for zlib's default" + end + if @source_format == :avro + raise Fluent::ConfigError, "compression gzip is not supported with source_format avro" + end + end + placeholder_params = "project=#{@project}/dataset=#{@dataset}/table=#{@tablelist.join(",")}/fetch_schema_table=#{@fetch_schema_table}" placeholder_validate!(:bigquery_load, placeholder_params) end @@ -201,7 +218,11 @@ def poll end end - def create_upload_source(chunk) + def create_upload_source(chunk, &block) + if @compression == :gzip + return create_gzip_upload_source(chunk, &block) + end + chunk_is_file = @buffer_config["@type"] == 'file' if chunk_is_file File.open(chunk.path) do |file| @@ -217,6 +238,29 @@ def create_upload_source(chunk) end end end + + def create_gzip_upload_source(chunk) + Tempfile.create("chunk-gzip-tmp") do |file| + file.binmode + started_at = Fluent::Clock.now + gz = Zlib::GzipWriter.new(file, @compression_level) + begin + chunk.write_to(gz) + raw_size = gz.pos + ensure + gz.finish + end + file.flush + compressed_size = file.size + log.debug("compressed chunk for load job", + chunk: dump_unique_id_hex(chunk.unique_id), + raw_bytes: raw_size, + compressed_bytes: compressed_size, + elapsed: (Fluent::Clock.now - started_at).round(3)) + file.rewind + yield file + end + end end end end diff --git a/test/plugin/test_out_bigquery_load.rb b/test/plugin/test_out_bigquery_load.rb index 66129fc..1607824 100644 --- a/test/plugin/test_out_bigquery_load.rb +++ b/test/plugin/test_out_bigquery_load.rb @@ -391,6 +391,75 @@ def test_write_with_auto_create_table_time_partitioning_expiration_in_seconds assert_equal({"type" => "DAY", "expirationMs" => "2592000000"}, JSON.parse(json)["configuration"]["load"]["timePartitioning"]) end + def test_configure_compression_default + driver = create_driver + assert_equal :none, driver.instance.compression + end + + def test_configure_compression_invalid_level + assert_raise(Fluent::ConfigError) do + create_driver(CONFIG + "compression gzip\ncompression_level 10\n") + end + end + + def test_configure_compression_gzip_with_avro + assert_raise(Fluent::ConfigError) do + create_driver(CONFIG + "compression gzip\nsource_format avro\n") + end + end + + data( + "memory buffer" => "", + "memory buffer with compress gzip" => "compress gzip", + ) + def test_write_with_compression_gzip(buffer_option) + driver = create_driver(<<-CONFIG) + table foo + email foo@bar.example + private_key_path /path/to/key + project yourproject_id + dataset yourdataset_id + + + @type memory + #{buffer_option} + + + schema_path #{SCHEMA_PATH} + wait_job_interval 0.1 + compression gzip + CONFIG + + uploaded = nil + response_stub = stub! + stub_writer do |writer| + mock(writer).fetch_load_job(is_a(Fluent::BigQuery::Writer::JobReference)) { response_stub } + mock(writer).commit_load_job(is_a(String), response_stub) + + mock(writer.client).get_table('yourproject_id', 'yourdataset_id', 'foo') { nil } + + mock(writer.client).insert_job('yourproject_id', anything, upload_source: duck_type(:read, :rewind), content_type: "application/octet-stream") do |_, _, options| + uploaded = options[:upload_source].read + stub!.job_reference.stub! do |s| + s.job_id { "dummy_job_id" } + s.location { "us" } + end + end + end + + driver.run do + driver.feed("tag", Time.now.to_i, {"a" => "b"}) + driver.feed("tag", Time.now.to_i, {"a" => "c"}) + end + + # The upload source must be a single gzip stream of newline delimited JSON + assert_equal "\x1f\x8b".b, uploaded.byteslice(0, 2) + reader = Zlib::GzipReader.new(StringIO.new(uploaded)) + lines = reader.read.each_line.map { |line| JSON.parse(line) } + assert_true reader.unused.nil? + assert_equal [{"a" => "b"}, {"a" => "c"}], lines + end + private def create_response_stub(response)