Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
132 changes: 132 additions & 0 deletions .github/actions/wait-for-ci/action.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,132 @@
name: "Wait for CI"
description: "Waits for all PR CI workflow runs to finish and verifies generator-tests succeeded when applicable"
inputs:
generator-changed:
required: false
default: "false"
description: "Whether generator files were modified in the PR"
runs:
using: "composite"
steps:
- name: Wait for all CI workflows to finish
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
env:
GENERATOR_CHANGED: ${{ inputs.generator-changed }}
with:
script: |
const owner = context.repo.owner;
const repo = context.repo.repo;
const headSha = context.payload.pull_request?.head?.sha || context.sha;
const generatorChanged = process.env.GENERATOR_CHANGED === 'true';
const pollIntervalMs = 30000;
const maxWaitMs = 46 * 60 * 1000;
const startTime = Date.now();
const sleep = ms => new Promise(resolve => setTimeout(resolve, ms));

const isPresubmitUnitWorkflow = run =>
run.path === '.github/workflows/presubmit.yaml' ||
run.path === '.github/workflows/windows-presubmit.yaml' ||
run.name === 'presubmit' ||
run.name === 'presubmit-windows';

const isGateJob = jobName =>
jobName === 'Wait for CI' || /^units \(\d+\)$/.test(jobName);

const isGeneratorWorkflow = run =>
run.path === '.github/workflows/generator-tests.yaml' ||
run.name === 'generator-tests';

while (Date.now() - startTime < maxWaitMs) {
Comment thread
shivanee-p marked this conversation as resolved.
try {
const { data: { workflow_runs: runs } } = await github.rest.actions.listWorkflowRunsForRepo({
owner,
repo,
head_sha: headSha,
event: 'pull_request',
per_page: 100,
});

// Keep only the most recent workflow run per workflow_id
const latestByWorkflow = new Map();
for (const run of runs) {
const existing = latestByWorkflow.get(run.workflow_id);
if (!existing || new Date(run.created_at) > new Date(existing.created_at)) {
latestByWorkflow.set(run.workflow_id, run);
}
}

const pendingItems = [];
let generatorRun = null;

for (const run of latestByWorkflow.values()) {
if (isGeneratorWorkflow(run)) {
generatorRun = run;
}

if (isPresubmitUnitWorkflow(run)) {
if (run.status === 'completed') {
continue;
}
const { data: { jobs } } = await github.rest.actions.listJobsForWorkflowRun({
owner,
repo,
run_id: run.id,
per_page: 100,
});
const workJobs = jobs.filter(j => !isGateJob(j.name));
if (workJobs.length === 0) {
pendingItems.push(`${run.name} (initializing jobs)`);
} else {
const runningJobs = workJobs.filter(j => j.status !== 'completed');
for (const j of runningJobs) {
pendingItems.push(`${run.name} / ${j.name} (${j.status})`);
}
}
} else if (run.status !== 'completed') {
pendingItems.push(`${run.name} (${run.status})`);
}
}

if (generatorChanged && !generatorRun) {
pendingItems.push('generator-tests (waiting for workflow run to register)');
}

if (pendingItems.length === 0) {
core.info(`All CI workflows for SHA ${headSha} have finished.`);
for (const run of latestByWorkflow.values()) {
if (isPresubmitUnitWorkflow(run)) {
continue;
}
core.info(`- ${run.name}: conclusion=${run.conclusion} (${run.html_url})`);
if (
!isGeneratorWorkflow(run) &&
run.conclusion !== 'success' &&
run.conclusion !== 'skipped' &&
run.conclusion !== 'neutral'
) {
core.warning(`Non-required workflow '${run.name}' finished with conclusion '${run.conclusion}': ${run.html_url}`);
}
}

if (generatorRun && generatorRun.conclusion !== 'success' && generatorRun.conclusion !== 'skipped') {
core.setFailed(`generator-tests finished with conclusion '${generatorRun.conclusion}': ${generatorRun.html_url}`);
return;
}
if (generatorChanged && (!generatorRun || generatorRun.conclusion !== 'success')) {
core.setFailed(
`generator-tests did not succeed (conclusion: ${generatorRun ? generatorRun.conclusion : 'missing'})`,
);
return;
}
return;
}

core.info(`Waiting for ${pendingItems.length} CI step(s) to finish:\n- ${pendingItems.join('\n- ')}`);
} catch (err) {
core.warning(`Error polling CI workflow status (will retry in ${pollIntervalMs / 1000}s): ${err instanceof Error ? err.message : String(err)}`);
}

await sleep(pollIntervalMs);
}

core.setFailed(`Timed out waiting for CI workflows to finish on SHA ${headSha}`);
55 changes: 50 additions & 5 deletions .github/workflows/generator-tests.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -10,18 +10,36 @@ on:
- 'core/generator/**'
- '.github/workflows/generator-tests.yaml'
pull_request:
paths:
- 'core/generator/**'
- '.github/workflows/generator-tests.yaml'
workflow_dispatch:

concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}

jobs:
presubmit:
if: github.event_name == 'pull_request'
changes:
runs-on: ubuntu-latest
outputs:
generator: ${{ steps.filter.outputs.generator }}
steps:
- uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1
with:
fetch-depth: 2
persist-credentials: false
- uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1
id: filter
with:
filters: |
generator:
- 'core/generator/**'
- '.github/workflows/generator-tests.yaml'

generator-tests:
needs: changes
if: >-
(github.event_name == 'pull_request' && needs.changes.outputs.generator == 'true') ||
github.event_name == 'push' ||
github.event_name == 'workflow_dispatch'
runs-on: ubuntu-latest
timeout-minutes: 45
container: gcr.io/gapic-images/googleapis:20250404
Expand Down Expand Up @@ -154,3 +172,30 @@ jobs:
echo "Some ESM library tests failed!"
exit 1
fi

# Consolidate generator test status to satisfy branch protection requirements
presubmit:
needs: [changes, generator-tests]
runs-on: ubuntu-latest
if: always()
env:
CHANGES_RESULT: ${{ needs.changes.result }}
GENERATOR_CHANGED: ${{ needs.changes.outputs.generator }}
TESTS_RESULT: ${{ needs.generator-tests.result }}
EVENT_NAME: ${{ github.event_name }}
steps:
- name: Check generator test status
run: |
if [ "$CHANGES_RESULT" != "success" ]; then
echo "Path filter job failed (result: $CHANGES_RESULT)."
exit 1
fi
if [ "$EVENT_NAME" = "pull_request" ] && [ "$GENERATOR_CHANGED" != "true" ]; then
echo "No generator changes detected; skipping generator tests."
exit 0
fi
if [ "$TESTS_RESULT" != "success" ]; then
echo "Generator tests did not succeed (result: $TESTS_RESULT)."
exit 1
fi
echo "Generator tests passed!"
33 changes: 32 additions & 1 deletion .github/workflows/presubmit.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,19 @@ jobs:
outputs:
shard-matrix: ${{ steps.set-matrix.outputs.shard_matrix }}
shard-total: ${{ steps.set-matrix.outputs.shard_total }}
generator-changed: ${{ steps.changes.outputs.generator }}
steps:
- uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0
with:
fetch-depth: 2
persist-credentials: false
- uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1
id: changes
with:
filters: |
generator:
- 'core/generator/**'
- '.github/workflows/generator-tests.yaml'
- id: set-matrix
uses: ./.github/actions/calculate-shard-matrix
units:
Expand Down Expand Up @@ -74,20 +82,43 @@ jobs:
env:
GIT_DIFF_ARG: "HEAD^1...HEAD -- :!packages"

wait-for-ci:
name: Wait for CI
needs: [setup, units, lint]
runs-on: ubuntu-latest
if: always()
timeout-minutes: 50
steps:
- uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0
with:
persist-credentials: false
- name: Wait for all CI workflows to finish
uses: ./.github/actions/wait-for-ci
with:
generator-changed: ${{ needs.setup.outputs.generator-changed }}

# Consolidate shards into jobs representing aggregate status to simplify branch protection requirements
units-status:
name: units (${{ matrix.node-version }})
needs: units
needs: [setup, units, wait-for-ci]
runs-on: ubuntu-latest
if: always()
strategy:
fail-fast: false
matrix:
node-version: [22, 24, 26]
env:
WAIT_FOR_CI_RESULT: ${{ needs.wait-for-ci.result }}
steps:
- uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0
with:
persist-credentials: false
- name: Verify CI completion and generator test status
run: |
if [ "$WAIT_FOR_CI_RESULT" != "success" ]; then
echo "CI completion check or generator-tests failed (result: $WAIT_FOR_CI_RESULT)."
exit 1
fi
- name: Check shard status
uses: ./.github/actions/check-shard-status
with:
Expand Down
33 changes: 32 additions & 1 deletion .github/workflows/windows-presubmit.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,19 @@ jobs:
outputs:
shard-matrix: ${{ steps.set-matrix.outputs.shard_matrix }}
shard-total: ${{ steps.set-matrix.outputs.shard_total }}
generator-changed: ${{ steps.changes.outputs.generator }}
steps:
- uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0
with:
fetch-depth: 2
persist-credentials: false
- uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1
id: changes
with:
filters: |
generator:
- 'core/generator/**'
- '.github/workflows/generator-tests.yaml'
- id: set-matrix
uses: ./.github/actions/calculate-shard-matrix
units:
Expand All @@ -42,20 +50,43 @@ jobs:
shard-total: ${{ needs.setup.outputs.shard-total }}
shard-index: ${{ matrix.shard-index }}

wait-for-ci:
name: Wait for CI
needs: [setup, units]
runs-on: ubuntu-latest
if: always()
timeout-minutes: 50
steps:
- uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0
with:
persist-credentials: false
- name: Wait for all CI workflows to finish
uses: ./.github/actions/wait-for-ci
with:
generator-changed: ${{ needs.setup.outputs.generator-changed }}

# Dummy jobs to satisfy branch protection requirements for each node version
units-status:
name: units (${{ matrix.node-version }})
needs: units
needs: [setup, units, wait-for-ci]
runs-on: ubuntu-latest
if: always()
strategy:
fail-fast: false
matrix:
node-version: [22, 24, 26]
env:
WAIT_FOR_CI_RESULT: ${{ needs.wait-for-ci.result }}
steps:
- uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0
with:
persist-credentials: false
- name: Verify CI completion and generator test status
run: |
if [ "$WAIT_FOR_CI_RESULT" != "success" ]; then
echo "CI completion check or generator-tests failed (result: $WAIT_FOR_CI_RESULT)."
exit 1
fi
- name: Check shard status
uses: ./.github/actions/check-shard-status
with:
Expand Down
21 changes: 12 additions & 9 deletions ci/run_conditional_tests.sh
Original file line number Diff line number Diff line change
Expand Up @@ -228,22 +228,22 @@ for subdir in "${subdirs[@]}"; do
if [[ "${changed}" -eq 0 ]]; then
echo "no change detected in ${d}, skipping"
else
if ([[ "${d}" == core/packages/* ]] || [[ "${d}" == core/dev-packages/* ]]) && [[ "${IS_CORE}" == "true" ]] && [[ "${TEST_TYPE}" == "system" ]]; then
if [[ ( "${d}" == core/packages/* || "${d}" == core/dev-packages/* ) && "${IS_CORE}" == "true" && "${TEST_TYPE}" == "system" ]]; then
echo "run system tests for core/packages in ${d}"
export RUN_INTERDEPENDENT_TESTS=true
should_test=true
elif ([[ "${d}" == core/packages/* ]] || [[ "${d}" == core/dev-packages/* ]]) && [[ "${IS_CORE}" == "true" ]] && [[ "${TEST_TYPE}" == "samples" ]]; then
elif [[ ( "${d}" == core/packages/* || "${d}" == core/dev-packages/* ) && "${IS_CORE}" == "true" && "${TEST_TYPE}" == "samples" ]]; then
if [[ "${tests_with_credentials[*]}" =~ "${d}" ]] && [[ -n "${GOOGLE_APPLICATION_CREDENTIALS}" ]]; then
echo "run samples tests with credentials for core/packages in ${d}"
should_test=true
elif ! [[ "${tests_with_credentials[*]}" =~ "${d}" ]] && [[ -z "${GOOGLE_APPLICATION_CREDENTIALS}" ]]; then
echo "run samples tests for core/packages in ${d}"
should_test=true
fi
elif [[ "${d}" == core/packages/* ]] && [[ "${JS_RUNTIME}" == "bun" || "${TEST_CMD}" == *bun* ]] && [[ "${TEST_TYPE}" == "units" ]]; then
echo "change detected in core package ${d} for Bun ${TEST_TYPE} test"
elif [[ ( "${d}" == core/packages/* || "${d}" == core/dev-packages/* ) && "${TEST_TYPE}" == "units" ]]; then
echo "change detected in core package ${d} for ${TEST_TYPE} test"
should_test=true
elif [[ "${d}" == core/packages/* ]] || [[ "${d}" == core/dev-packages/* ]]; then
elif [[ "${d}" == core/packages/* || "${d}" == core/dev-packages/* ]]; then
echo "skipping core package ${d} in non-core trigger"
elif [[ "${TEST_TYPE}" == "system" ]] || [[ "${TEST_TYPE}" == "lint" ]] || [[ "${TEST_TYPE}" == "units" ]]; then
echo "change detected in ${d} for ${TEST_TYPE} test"
Expand All @@ -258,20 +258,23 @@ for subdir in "${subdirs[@]}"; do
fi
else
# If GIT_DIFF_ARG is empty, run all the tests.
if ([[ "${d}" == core/packages/* ]] || [[ "${d}" == core/dev-packages/* ]]) && [[ "${IS_CORE}" == "true" ]] && [[ "${TEST_TYPE}" == "system" ]]; then
if [[ ( "${d}" == core/packages/* || "${d}" == core/dev-packages/* ) && "${IS_CORE}" == "true" && "${TEST_TYPE}" == "system" ]]; then
echo "run system tests for core/packages in ${d}"
export RUN_INTERDEPENDENT_TESTS=true
should_test=true
elif ([[ "${d}" == core/packages/* ]] || [[ "${d}" == core/dev-packages/* ]]) && [[ "${IS_CORE}" == "true" ]] && [[ "${TEST_TYPE}" == "samples" ]]; then
elif [[ ( "${d}" == core/packages/* || "${d}" == core/dev-packages/* ) && "${IS_CORE}" == "true" && "${TEST_TYPE}" == "samples" ]]; then
if [[ "${tests_with_credentials[*]}" =~ "${d}" ]] && [[ -n "${GOOGLE_APPLICATION_CREDENTIALS}" ]]; then
echo "run samples tests with credentials for core/packages in ${d}"
should_test=true
elif ! [[ "${tests_with_credentials[*]}" =~ "${d}" ]] && [[ -z "${GOOGLE_APPLICATION_CREDENTIALS}" ]]; then
echo "run samples tests for core/packages in ${d}"
should_test=true
fi
# elif [[ "${d}" == core/packages/* ]] || [[ "${d}" == core/dev-packages/* ]]; then
# echo "skipping core package ${d} in non-core trigger"
elif [[ ( "${d}" == core/packages/* || "${d}" == core/dev-packages/* ) && "${TEST_TYPE}" == "units" ]]; then
echo "run ${TEST_TYPE} test for core package ${d}"
should_test=true
elif [[ "${d}" == core/packages/* || "${d}" == core/dev-packages/* ]]; then
echo "skipping core package ${d} in non-core trigger"
elif [[ "${TEST_TYPE}" == "system" ]] || [[ "${TEST_TYPE}" == "lint" ]] || [[ "${TEST_TYPE}" == "units" ]]; then
echo "run ${TEST_TYPE} test for ${d}"
should_test=true
Expand Down
Loading