From 2f8767224636fb963b42717c3ae4b23ee579e905 Mon Sep 17 00:00:00 2001 From: Lucas Barros Date: Thu, 12 Nov 2026 01:59:12 -0300 Subject: [PATCH 1/2] Send time_partitioning_expiration as an integer number of milliseconds time_partitioning_expiration is a Fluentd time parameter, so a plain number such as 2592000 is parsed as a Float. Multiplied by 1000 it became 2592000000.0, which the client serializes as "2592000000.0", and BigQuery rejects the load job or the table creation with "Invalid value at 'job.configuration.load.time_partitioning.expiration_ms.value' (TYPE_INT64)". Values with a unit such as 30d were Integers and worked. Convert the milliseconds to an Integer, and say in the README that the parameter takes a time value and is sent in milliseconds. Signed-off-by: Lucas Barros --- README.md | 2 +- lib/fluent/plugin/bigquery/writer.rb | 2 +- test/plugin/test_out_bigquery_load.rb | 44 +++++++++++++++++++++++++++ 3 files changed, 46 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index c2d4ac8..9e95f1f 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,7 @@ Because embedded gem dependency sometimes restricts ruby environment. | request_open_timeout_sec | integer | no | no | 60 | Bigquery API connection, and request timeout. If you send big data to Bigquery, set large value. | | time_partitioning_type | enum | no (either day or hour) | no | nil | Type of bigquery time partitioning feature. | | time_partitioning_field | string | no | no | nil | Field used to determine how to create a time-based partition. | -| time_partitioning_expiration | time | no | no | nil | Expiration milliseconds for bigquery time partitioning. | +| time_partitioning_expiration | time | no | no | nil | Expiration of each partition, as a time value (`2592000`, `720h`, `30d`). It is sent to BigQuery in milliseconds. | | clustering_fields | array(string) | no | no | nil | One or more fields on which data should be clustered. The order of the specified columns determines the sort order of the data. | #### bigquery_insert diff --git a/lib/fluent/plugin/bigquery/writer.rb b/lib/fluent/plugin/bigquery/writer.rb index 20a6335..3198534 100644 --- a/lib/fluent/plugin/bigquery/writer.rb +++ b/lib/fluent/plugin/bigquery/writer.rb @@ -313,7 +313,7 @@ def time_partitioning @time_partitioning = { type: @options[:time_partitioning_type].to_s.upcase, field: @options[:time_partitioning_field] ? @options[:time_partitioning_field].to_s : nil, - expiration_ms: @options[:time_partitioning_expiration] ? @options[:time_partitioning_expiration] * 1000 : nil, + expiration_ms: @options[:time_partitioning_expiration] ? (@options[:time_partitioning_expiration] * 1000).to_i : nil, }.reject { |_, v| v.nil? } else @time_partitioning diff --git a/test/plugin/test_out_bigquery_load.rb b/test/plugin/test_out_bigquery_load.rb index 1c5d225..fcea136 100644 --- a/test/plugin/test_out_bigquery_load.rb +++ b/test/plugin/test_out_bigquery_load.rb @@ -342,6 +342,50 @@ def test_write_with_auto_create_table end end + def test_write_with_auto_create_table_time_partitioning_expiration_in_seconds + driver = create_driver(<<-CONFIG) + table foo + email foo@bar.example + private_key_path /path/to/key + project yourproject_id + dataset yourdataset_id + + + @type memory + + + auto_create_table true + schema_path #{SCHEMA_PATH} + + time_partitioning_type day + time_partitioning_expiration 2592000 + CONFIG + + job = nil + stub_writer do |writer| + mock(writer.client).get_table('yourproject_id', 'yourdataset_id', 'foo') do + raise Google::Apis::ClientError.new("notFound: Not found: Table yourproject_id:yourdataset_id.foo", status_code: 404) + end + + mock(writer.client).insert_job('yourproject_id', anything, upload_source: duck_type(:write, :sync, :rewind), content_type: "application/octet-stream") do |_, configuration, _| + job = configuration + stub!.job_reference.stub! do |s| + s.job_id { "dummy_job_id" } + s.location { "us" } + end + end + end + + driver.run do + driver.feed("tag", Time.now.to_i, {"a" => "b"}) + end + + time_partitioning = job[:configuration][:load][:time_partitioning] + assert_equal({type: "DAY", expiration_ms: 2592000000}, time_partitioning) + json = Google::Apis::BigqueryV2::Job::Representation.new(job).to_json(user_options: { skip_undefined: true }) + assert_equal({"type" => "DAY", "expirationMs" => "2592000000"}, JSON.parse(json)["configuration"]["load"]["timePartitioning"]) + end + private def create_response_stub(response) From f02fa82f272db6abec4a5ad93e7a2760fab6b2b9 Mon Sep 17 00:00:00 2001 From: Lucas Barros Date: Fri, 13 Nov 2026 20:56:23 -0300 Subject: [PATCH 2/2] Mock the load job polling in the expiration test and document the unit A plain time_partitioning_expiration is in seconds. The test now mocks fetch_load_job and commit_load_job so it runs without network access, and checks that expiration_ms is an Integer. Signed-off-by: Lucas Barros --- README.md | 2 +- test/plugin/test_out_bigquery_load.rb | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 9e95f1f..190a884 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,7 @@ Because embedded gem dependency sometimes restricts ruby environment. | request_open_timeout_sec | integer | no | no | 60 | Bigquery API connection, and request timeout. If you send big data to Bigquery, set large value. | | time_partitioning_type | enum | no (either day or hour) | no | nil | Type of bigquery time partitioning feature. | | time_partitioning_field | string | no | no | nil | Field used to determine how to create a time-based partition. | -| time_partitioning_expiration | time | no | no | nil | Expiration of each partition, as a time value (`2592000`, `720h`, `30d`). It is sent to BigQuery in milliseconds. | +| time_partitioning_expiration | time | no | no | nil | Expiration of each partition, as a time value (`2592000`, `720h`, `30d`). A plain number is in seconds. It is sent to BigQuery in milliseconds. | | clustering_fields | array(string) | no | no | nil | One or more fields on which data should be clustered. The order of the specified columns determines the sort order of the data. | #### bigquery_insert diff --git a/test/plugin/test_out_bigquery_load.rb b/test/plugin/test_out_bigquery_load.rb index fcea136..66129fc 100644 --- a/test/plugin/test_out_bigquery_load.rb +++ b/test/plugin/test_out_bigquery_load.rb @@ -362,7 +362,11 @@ def test_write_with_auto_create_table_time_partitioning_expiration_in_seconds CONFIG job = nil + response_stub = stub! stub_writer do |writer| + mock(writer).fetch_load_job(is_a(Fluent::BigQuery::Writer::JobReference)) { response_stub } + mock(writer).commit_load_job(is_a(String), response_stub) + mock(writer.client).get_table('yourproject_id', 'yourdataset_id', 'foo') do raise Google::Apis::ClientError.new("notFound: Not found: Table yourproject_id:yourdataset_id.foo", status_code: 404) end @@ -382,6 +386,7 @@ def test_write_with_auto_create_table_time_partitioning_expiration_in_seconds time_partitioning = job[:configuration][:load][:time_partitioning] assert_equal({type: "DAY", expiration_ms: 2592000000}, time_partitioning) + assert_kind_of(Integer, time_partitioning[:expiration_ms]) json = Google::Apis::BigqueryV2::Job::Representation.new(job).to_json(user_options: { skip_undefined: true }) assert_equal({"type" => "DAY", "expirationMs" => "2592000000"}, JSON.parse(json)["configuration"]["load"]["timePartitioning"]) end