From 60b366ded9e5af24ad1d201105605842acec7cb2 Mon Sep 17 00:00:00 2001 From: Lucas Barros Date: Tue, 17 Nov 2026 01:08:03 -0300 Subject: [PATCH 1/2] Use the dataset location for a duplicate load job when the table is missing With prevent_duplicate_load, a 409 from jobs.insert is resolved by polling the existing job in the table's location. When auto_create_table is on and the table does not exist yet (the duplicate job is the one creating it), get_table raised 404 and left table nil, so table.location raised NoMethodError and the chunk was retried forever. Fall back to the dataset location, which is where the table and the load job live. Signed-off-by: Lucas Barros --- lib/fluent/plugin/bigquery/writer.rb | 4 ++- test/plugin/test_out_bigquery_load.rb | 43 +++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/lib/fluent/plugin/bigquery/writer.rb b/lib/fluent/plugin/bigquery/writer.rb index 3198534..1e28347 100644 --- a/lib/fluent/plugin/bigquery/writer.rb +++ b/lib/fluent/plugin/bigquery/writer.rb @@ -169,7 +169,9 @@ def create_load_job(chunk_id, chunk_id_hex, project, dataset, table_id, upload_s if job_id && e.status_code == 409 && e.message =~ /Job/ # duplicate load job # If a load job is duplicated, the API response may not be available to create the result. # Therefore, we need to use the location of the table instead of the job's location to determine the result. - return JobReference.new(chunk_id, chunk_id_hex, project, dataset, table_id, job_id, table.location) + # The table is missing when the duplicated job is the one auto-creating it, so fall back to the dataset location. + location = table ? table.location : client.get_dataset(project, dataset).location + return JobReference.new(chunk_id, chunk_id_hex, project, dataset, table_id, job_id, location) end raise Fluent::BigQuery::Error.wrap(e) diff --git a/test/plugin/test_out_bigquery_load.rb b/test/plugin/test_out_bigquery_load.rb index 66129fc..0bc8d75 100644 --- a/test/plugin/test_out_bigquery_load.rb +++ b/test/plugin/test_out_bigquery_load.rb @@ -133,6 +133,49 @@ def test_write_with_prevent_duplicate_load end end + def test_write_with_prevent_duplicate_load_while_auto_creating_table + driver = create_driver(<<-CONFIG) + table foo + email foo@bar.example + private_key_path /path/to/key + project yourproject_id + dataset yourdataset_id + + + @type memory + + + + time_format %s + time_key time + + + schema_path #{SCHEMA_PATH} + auto_create_table true + prevent_duplicate_load true + CONFIG + + response_stub = stub! + stub_writer do |writer| + mock(writer.client).get_table('yourproject_id', 'yourdataset_id', 'foo') do + raise Google::Apis::ClientError.new("notFound: Not found: Table yourproject_id:yourdataset_id.foo", status_code: 404) + end + mock(writer.client).insert_job.with_any_args do + raise Google::Apis::ClientError.new("duplicate: Already Exists: Job yourproject_id:asia-northeast1.fluentd_job_x", status_code: 409) + end + mock(writer.client).get_dataset('yourproject_id', 'yourdataset_id') do + stub!.location { "asia-northeast1" } + end + + mock(writer).fetch_load_job(satisfy { |ref| ref.job_id =~ /\Afluentd_job_/ && ref.location == "asia-northeast1" }) { response_stub } + mock(writer).commit_load_job(is_a(String), response_stub) + end + + driver.run do + driver.feed("tag", Time.now.to_i, {"a" => "b"}) + end + end + def test_write_with_retryable_error driver = create_driver From 0ae288d46cfcc2010034e0fe644c8f97b3a58d55 Mon Sep 17 00:00:00 2001 From: Lucas Barros Date: Tue, 17 Nov 2026 19:45:12 -0300 Subject: [PATCH 2/2] Wrap datasets.get errors for a duplicate load job Signed-off-by: Lucas Barros --- lib/fluent/plugin/bigquery/writer.rb | 9 ++++- test/plugin/test_out_bigquery_load.rb | 54 +++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 1 deletion(-) diff --git a/lib/fluent/plugin/bigquery/writer.rb b/lib/fluent/plugin/bigquery/writer.rb index 1e28347..75d3a4b 100644 --- a/lib/fluent/plugin/bigquery/writer.rb +++ b/lib/fluent/plugin/bigquery/writer.rb @@ -170,7 +170,7 @@ def create_load_job(chunk_id, chunk_id_hex, project, dataset, table_id, upload_s # If a load job is duplicated, the API response may not be available to create the result. # Therefore, we need to use the location of the table instead of the job's location to determine the result. # The table is missing when the duplicated job is the one auto-creating it, so fall back to the dataset location. - location = table ? table.location : client.get_dataset(project, dataset).location + location = table ? table.location : dataset_location(project, dataset) return JobReference.new(chunk_id, chunk_id_hex, project, dataset, table_id, job_id, location) end @@ -285,6 +285,13 @@ def get_auth_from_application_default Google::Auth.get_application_default([@scope]) end + def dataset_location(project, dataset) + client.get_dataset(project, dataset).location + rescue Google::Apis::ServerError, Google::Apis::ClientError, Google::Apis::AuthorizationError => e + log.error "datasets.get API", project_id: project, dataset: dataset, code: e.status_code, message: e.message + raise Fluent::BigQuery::Error.wrap(e) + end + def safe_table_id(table_id) table_id.gsub(/\$\d+$/, "") end diff --git a/test/plugin/test_out_bigquery_load.rb b/test/plugin/test_out_bigquery_load.rb index 0bc8d75..77babc7 100644 --- a/test/plugin/test_out_bigquery_load.rb +++ b/test/plugin/test_out_bigquery_load.rb @@ -327,6 +327,60 @@ def test_write_with_not_retryable_error driver.instance_shutdown end + def test_write_with_not_retryable_error_from_dataset_location + driver = create_driver(<<-CONFIG) + table foo + email foo@bar.example + private_key_path /path/to/key + project yourproject_id + dataset yourdataset_id + + + @type memory + + + + time_format %s + time_key time + + + schema_path #{SCHEMA_PATH} + auto_create_table true + prevent_duplicate_load true + + @type file + path error + utc + + CONFIG + + driver.instance_start + tag, time, record = "tag", Time.now.to_i, {"a" => "b"} + metadata = Fluent::Plugin::Buffer::Metadata.new(tag, time, record) + chunk = driver.instance.buffer.generate_chunk(metadata).tap do |c| + c.append([driver.instance.format(tag, time, record)]) + end + + stub_writer do |writer| + mock(writer.client).get_table('yourproject_id', 'yourdataset_id', 'foo') do + raise Google::Apis::ClientError.new("notFound: Not found: Table yourproject_id:yourdataset_id.foo", status_code: 404) + end + mock(writer.client).insert_job.with_any_args do + raise Google::Apis::ClientError.new("duplicate: Already Exists: Job yourproject_id:asia-northeast1.fluentd_job_x", status_code: 409) + end + mock(writer.client).get_dataset('yourproject_id', 'yourdataset_id') do + raise Google::Apis::ClientError.new("accessDenied: Access Denied: Dataset yourproject_id:yourdataset_id", status_code: 403) + end + end + + error = assert_raise Fluent::BigQuery::UnRetryableError do + driver.instance.write(chunk) + end + assert_equal 403, error.status_code + assert_in_delta driver.instance.retry.secondary_transition_at , Time.now, 0.1 + driver.instance_shutdown + end + def test_write_with_auto_create_table driver = create_driver(<<-CONFIG) table foo