diff --git a/lib/fluent/plugin/bigquery/writer.rb b/lib/fluent/plugin/bigquery/writer.rb
index 3198534..75d3a4b 100644
--- a/lib/fluent/plugin/bigquery/writer.rb
+++ b/lib/fluent/plugin/bigquery/writer.rb
@@ -169,7 +169,9 @@ def create_load_job(chunk_id, chunk_id_hex, project, dataset, table_id, upload_s
if job_id && e.status_code == 409 && e.message =~ /Job/ # duplicate load job
# If a load job is duplicated, the API response may not be available to create the result.
# Therefore, we need to use the location of the table instead of the job's location to determine the result.
- return JobReference.new(chunk_id, chunk_id_hex, project, dataset, table_id, job_id, table.location)
+ # The table is missing when the duplicated job is the one auto-creating it, so fall back to the dataset location.
+ location = table ? table.location : dataset_location(project, dataset)
+ return JobReference.new(chunk_id, chunk_id_hex, project, dataset, table_id, job_id, location)
end
raise Fluent::BigQuery::Error.wrap(e)
@@ -283,6 +285,13 @@ def get_auth_from_application_default
Google::Auth.get_application_default([@scope])
end
+ def dataset_location(project, dataset)
+ client.get_dataset(project, dataset).location
+ rescue Google::Apis::ServerError, Google::Apis::ClientError, Google::Apis::AuthorizationError => e
+ log.error "datasets.get API", project_id: project, dataset: dataset, code: e.status_code, message: e.message
+ raise Fluent::BigQuery::Error.wrap(e)
+ end
+
def safe_table_id(table_id)
table_id.gsub(/\$\d+$/, "")
end
diff --git a/test/plugin/test_out_bigquery_load.rb b/test/plugin/test_out_bigquery_load.rb
index 66129fc..77babc7 100644
--- a/test/plugin/test_out_bigquery_load.rb
+++ b/test/plugin/test_out_bigquery_load.rb
@@ -133,6 +133,49 @@ def test_write_with_prevent_duplicate_load
end
end
+ def test_write_with_prevent_duplicate_load_while_auto_creating_table
+ driver = create_driver(<<-CONFIG)
+ table foo
+ email foo@bar.example
+ private_key_path /path/to/key
+ project yourproject_id
+ dataset yourdataset_id
+
+
+ @type memory
+
+
+
+ time_format %s
+ time_key time
+
+
+ schema_path #{SCHEMA_PATH}
+ auto_create_table true
+ prevent_duplicate_load true
+ CONFIG
+
+ response_stub = stub!
+ stub_writer do |writer|
+ mock(writer.client).get_table('yourproject_id', 'yourdataset_id', 'foo') do
+ raise Google::Apis::ClientError.new("notFound: Not found: Table yourproject_id:yourdataset_id.foo", status_code: 404)
+ end
+ mock(writer.client).insert_job.with_any_args do
+ raise Google::Apis::ClientError.new("duplicate: Already Exists: Job yourproject_id:asia-northeast1.fluentd_job_x", status_code: 409)
+ end
+ mock(writer.client).get_dataset('yourproject_id', 'yourdataset_id') do
+ stub!.location { "asia-northeast1" }
+ end
+
+ mock(writer).fetch_load_job(satisfy { |ref| ref.job_id =~ /\Afluentd_job_/ && ref.location == "asia-northeast1" }) { response_stub }
+ mock(writer).commit_load_job(is_a(String), response_stub)
+ end
+
+ driver.run do
+ driver.feed("tag", Time.now.to_i, {"a" => "b"})
+ end
+ end
+
def test_write_with_retryable_error
driver = create_driver
@@ -284,6 +327,60 @@ def test_write_with_not_retryable_error
driver.instance_shutdown
end
+ def test_write_with_not_retryable_error_from_dataset_location
+ driver = create_driver(<<-CONFIG)
+ table foo
+ email foo@bar.example
+ private_key_path /path/to/key
+ project yourproject_id
+ dataset yourdataset_id
+
+
+ @type memory
+
+
+
+ time_format %s
+ time_key time
+
+
+ schema_path #{SCHEMA_PATH}
+ auto_create_table true
+ prevent_duplicate_load true
+
+ @type file
+ path error
+ utc
+
+ CONFIG
+
+ driver.instance_start
+ tag, time, record = "tag", Time.now.to_i, {"a" => "b"}
+ metadata = Fluent::Plugin::Buffer::Metadata.new(tag, time, record)
+ chunk = driver.instance.buffer.generate_chunk(metadata).tap do |c|
+ c.append([driver.instance.format(tag, time, record)])
+ end
+
+ stub_writer do |writer|
+ mock(writer.client).get_table('yourproject_id', 'yourdataset_id', 'foo') do
+ raise Google::Apis::ClientError.new("notFound: Not found: Table yourproject_id:yourdataset_id.foo", status_code: 404)
+ end
+ mock(writer.client).insert_job.with_any_args do
+ raise Google::Apis::ClientError.new("duplicate: Already Exists: Job yourproject_id:asia-northeast1.fluentd_job_x", status_code: 409)
+ end
+ mock(writer.client).get_dataset('yourproject_id', 'yourdataset_id') do
+ raise Google::Apis::ClientError.new("accessDenied: Access Denied: Dataset yourproject_id:yourdataset_id", status_code: 403)
+ end
+ end
+
+ error = assert_raise Fluent::BigQuery::UnRetryableError do
+ driver.instance.write(chunk)
+ end
+ assert_equal 403, error.status_code
+ assert_in_delta driver.instance.retry.secondary_transition_at , Time.now, 0.1
+ driver.instance_shutdown
+ end
+
def test_write_with_auto_create_table
driver = create_driver(<<-CONFIG)
table foo