diff --git a/lib/fluent/plugin/bigquery/writer.rb b/lib/fluent/plugin/bigquery/writer.rb index 3198534..75d3a4b 100644 --- a/lib/fluent/plugin/bigquery/writer.rb +++ b/lib/fluent/plugin/bigquery/writer.rb @@ -169,7 +169,9 @@ def create_load_job(chunk_id, chunk_id_hex, project, dataset, table_id, upload_s if job_id && e.status_code == 409 && e.message =~ /Job/ # duplicate load job # If a load job is duplicated, the API response may not be available to create the result. # Therefore, we need to use the location of the table instead of the job's location to determine the result. - return JobReference.new(chunk_id, chunk_id_hex, project, dataset, table_id, job_id, table.location) + # The table is missing when the duplicated job is the one auto-creating it, so fall back to the dataset location. + location = table ? table.location : dataset_location(project, dataset) + return JobReference.new(chunk_id, chunk_id_hex, project, dataset, table_id, job_id, location) end raise Fluent::BigQuery::Error.wrap(e) @@ -283,6 +285,13 @@ def get_auth_from_application_default Google::Auth.get_application_default([@scope]) end + def dataset_location(project, dataset) + client.get_dataset(project, dataset).location + rescue Google::Apis::ServerError, Google::Apis::ClientError, Google::Apis::AuthorizationError => e + log.error "datasets.get API", project_id: project, dataset: dataset, code: e.status_code, message: e.message + raise Fluent::BigQuery::Error.wrap(e) + end + def safe_table_id(table_id) table_id.gsub(/\$\d+$/, "") end diff --git a/test/plugin/test_out_bigquery_load.rb b/test/plugin/test_out_bigquery_load.rb index 66129fc..77babc7 100644 --- a/test/plugin/test_out_bigquery_load.rb +++ b/test/plugin/test_out_bigquery_load.rb @@ -133,6 +133,49 @@ def test_write_with_prevent_duplicate_load end end + def test_write_with_prevent_duplicate_load_while_auto_creating_table + driver = create_driver(<<-CONFIG) + table foo + email foo@bar.example + private_key_path /path/to/key + project yourproject_id + dataset yourdataset_id + + + @type memory + + + + time_format %s + time_key time + + + schema_path #{SCHEMA_PATH} + auto_create_table true + prevent_duplicate_load true + CONFIG + + response_stub = stub! + stub_writer do |writer| + mock(writer.client).get_table('yourproject_id', 'yourdataset_id', 'foo') do + raise Google::Apis::ClientError.new("notFound: Not found: Table yourproject_id:yourdataset_id.foo", status_code: 404) + end + mock(writer.client).insert_job.with_any_args do + raise Google::Apis::ClientError.new("duplicate: Already Exists: Job yourproject_id:asia-northeast1.fluentd_job_x", status_code: 409) + end + mock(writer.client).get_dataset('yourproject_id', 'yourdataset_id') do + stub!.location { "asia-northeast1" } + end + + mock(writer).fetch_load_job(satisfy { |ref| ref.job_id =~ /\Afluentd_job_/ && ref.location == "asia-northeast1" }) { response_stub } + mock(writer).commit_load_job(is_a(String), response_stub) + end + + driver.run do + driver.feed("tag", Time.now.to_i, {"a" => "b"}) + end + end + def test_write_with_retryable_error driver = create_driver @@ -284,6 +327,60 @@ def test_write_with_not_retryable_error driver.instance_shutdown end + def test_write_with_not_retryable_error_from_dataset_location + driver = create_driver(<<-CONFIG) + table foo + email foo@bar.example + private_key_path /path/to/key + project yourproject_id + dataset yourdataset_id + + + @type memory + + + + time_format %s + time_key time + + + schema_path #{SCHEMA_PATH} + auto_create_table true + prevent_duplicate_load true + + @type file + path error + utc + + CONFIG + + driver.instance_start + tag, time, record = "tag", Time.now.to_i, {"a" => "b"} + metadata = Fluent::Plugin::Buffer::Metadata.new(tag, time, record) + chunk = driver.instance.buffer.generate_chunk(metadata).tap do |c| + c.append([driver.instance.format(tag, time, record)]) + end + + stub_writer do |writer| + mock(writer.client).get_table('yourproject_id', 'yourdataset_id', 'foo') do + raise Google::Apis::ClientError.new("notFound: Not found: Table yourproject_id:yourdataset_id.foo", status_code: 404) + end + mock(writer.client).insert_job.with_any_args do + raise Google::Apis::ClientError.new("duplicate: Already Exists: Job yourproject_id:asia-northeast1.fluentd_job_x", status_code: 409) + end + mock(writer.client).get_dataset('yourproject_id', 'yourdataset_id') do + raise Google::Apis::ClientError.new("accessDenied: Access Denied: Dataset yourproject_id:yourdataset_id", status_code: 403) + end + end + + error = assert_raise Fluent::BigQuery::UnRetryableError do + driver.instance.write(chunk) + end + assert_equal 403, error.status_code + assert_in_delta driver.instance.retry.secondary_transition_at , Time.now, 0.1 + driver.instance_shutdown + end + def test_write_with_auto_create_table driver = create_driver(<<-CONFIG) table foo