Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 10 additions & 1 deletion lib/fluent/plugin/bigquery/writer.rb
Original file line number Diff line number Diff line change
Expand Up @@ -169,7 +169,9 @@ def create_load_job(chunk_id, chunk_id_hex, project, dataset, table_id, upload_s
if job_id && e.status_code == 409 && e.message =~ /Job/ # duplicate load job
# If a load job is duplicated, the API response may not be available to create the result.
# Therefore, we need to use the location of the table instead of the job's location to determine the result.
return JobReference.new(chunk_id, chunk_id_hex, project, dataset, table_id, job_id, table.location)
# The table is missing when the duplicated job is the one auto-creating it, so fall back to the dataset location.
location = table ? table.location : dataset_location(project, dataset)
return JobReference.new(chunk_id, chunk_id_hex, project, dataset, table_id, job_id, location)
end

raise Fluent::BigQuery::Error.wrap(e)
Expand Down Expand Up @@ -283,6 +285,13 @@ def get_auth_from_application_default
Google::Auth.get_application_default([@scope])
end

def dataset_location(project, dataset)
client.get_dataset(project, dataset).location
rescue Google::Apis::ServerError, Google::Apis::ClientError, Google::Apis::AuthorizationError => e
log.error "datasets.get API", project_id: project, dataset: dataset, code: e.status_code, message: e.message
raise Fluent::BigQuery::Error.wrap(e)
end

def safe_table_id(table_id)
table_id.gsub(/\$\d+$/, "")
end
Expand Down
97 changes: 97 additions & 0 deletions test/plugin/test_out_bigquery_load.rb
Original file line number Diff line number Diff line change
Expand Up @@ -133,6 +133,49 @@ def test_write_with_prevent_duplicate_load
end
end

def test_write_with_prevent_duplicate_load_while_auto_creating_table
driver = create_driver(<<-CONFIG)
table foo
email foo@bar.example
private_key_path /path/to/key
project yourproject_id
dataset yourdataset_id

<buffer>
@type memory
</buffer>

<inject>
time_format %s
time_key time
</inject>

schema_path #{SCHEMA_PATH}
auto_create_table true
prevent_duplicate_load true
CONFIG

response_stub = stub!
stub_writer do |writer|
mock(writer.client).get_table('yourproject_id', 'yourdataset_id', 'foo') do
raise Google::Apis::ClientError.new("notFound: Not found: Table yourproject_id:yourdataset_id.foo", status_code: 404)
end
mock(writer.client).insert_job.with_any_args do
raise Google::Apis::ClientError.new("duplicate: Already Exists: Job yourproject_id:asia-northeast1.fluentd_job_x", status_code: 409)
end
mock(writer.client).get_dataset('yourproject_id', 'yourdataset_id') do
stub!.location { "asia-northeast1" }
end

mock(writer).fetch_load_job(satisfy { |ref| ref.job_id =~ /\Afluentd_job_/ && ref.location == "asia-northeast1" }) { response_stub }
mock(writer).commit_load_job(is_a(String), response_stub)
end

driver.run do
driver.feed("tag", Time.now.to_i, {"a" => "b"})
end
end

def test_write_with_retryable_error
driver = create_driver

Expand Down Expand Up @@ -284,6 +327,60 @@ def test_write_with_not_retryable_error
driver.instance_shutdown
end

def test_write_with_not_retryable_error_from_dataset_location
driver = create_driver(<<-CONFIG)
table foo
email foo@bar.example
private_key_path /path/to/key
project yourproject_id
dataset yourdataset_id

<buffer>
@type memory
</buffer>

<inject>
time_format %s
time_key time
</inject>

schema_path #{SCHEMA_PATH}
auto_create_table true
prevent_duplicate_load true
<secondary>
@type file
path error
utc
</secondary>
CONFIG

driver.instance_start
tag, time, record = "tag", Time.now.to_i, {"a" => "b"}
metadata = Fluent::Plugin::Buffer::Metadata.new(tag, time, record)
chunk = driver.instance.buffer.generate_chunk(metadata).tap do |c|
c.append([driver.instance.format(tag, time, record)])
end

stub_writer do |writer|
mock(writer.client).get_table('yourproject_id', 'yourdataset_id', 'foo') do
raise Google::Apis::ClientError.new("notFound: Not found: Table yourproject_id:yourdataset_id.foo", status_code: 404)
end
mock(writer.client).insert_job.with_any_args do
raise Google::Apis::ClientError.new("duplicate: Already Exists: Job yourproject_id:asia-northeast1.fluentd_job_x", status_code: 409)
end
mock(writer.client).get_dataset('yourproject_id', 'yourdataset_id') do
raise Google::Apis::ClientError.new("accessDenied: Access Denied: Dataset yourproject_id:yourdataset_id", status_code: 403)
end
end

error = assert_raise Fluent::BigQuery::UnRetryableError do
driver.instance.write(chunk)
end
assert_equal 403, error.status_code
assert_in_delta driver.instance.retry.secondary_transition_at , Time.now, 0.1
driver.instance_shutdown
end

def test_write_with_auto_create_table
driver = create_driver(<<-CONFIG)
table foo
Expand Down
Loading