From bda3ca5960b0bba7d43386c30c602e71a61f6a7c Mon Sep 17 00:00:00 2001 From: Andrea Cosentino Date: Fri, 24 Jul 2026 11:37:00 +0200 Subject: [PATCH] CAMEL-24252: Fix the container upgrade workflow aborting the PR loop MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit update-metadata-version.py exited 1 whenever no metadata entry matched, and the workflow step runs under bash -e, so the first container whose version has no serviceVersion counterpart aborted the whole loop: the remaining containers never got a PR and the job went red. Match the entries the way CamelTestInfraGenerateMetadataMojo derives serviceVersion — the property key prefix against the entry aliases — instead of the module directory name, so a container.properties shared across modules (azure.container, owned by camel-test-infra-azure-common but feeding the azure-storage-blob and azure-storage-queue entries) is resolved. Platform-specific keys, version filters and properties with no metadata counterpart are now reported as a no-op instead of an error, and the workflow keeps going if the update fails anyway. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Andrea Cosentino --- .../test_update_metadata_version.py | 207 ++++++++++++++++++ .../update-metadata-version.py | 110 ++++++++-- .../workflows/check-container-versions.yml | 12 +- .../workflows/pr-ci-scripts-validation.yml | 47 ++++ .gitignore | 1 + 5 files changed, 350 insertions(+), 27 deletions(-) create mode 100644 .github/actions/check-container-upgrade/test_update_metadata_version.py create mode 100644 .github/workflows/pr-ci-scripts-validation.yml diff --git a/.github/actions/check-container-upgrade/test_update_metadata_version.py b/.github/actions/check-container-upgrade/test_update_metadata_version.py new file mode 100644 index 0000000000000..4b0db7ecd18d2 --- /dev/null +++ b/.github/actions/check-container-upgrade/test_update_metadata_version.py @@ -0,0 +1,207 @@ +#!/usr/bin/env python3 +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Tests for update-metadata-version.py. Run with: python3 -m unittest discover""" + +import contextlib +import importlib.util +import io +import json +import os +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +HERE = Path(__file__).parent +REPO_ROOT = HERE.parents[2] +METADATA_INFRA = REPO_ROOT / "test-infra/camel-test-infra-all/src/generated/resources/META-INF/metadata.json" + +_spec = importlib.util.spec_from_file_location("update_metadata_version", HERE / "update-metadata-version.py") +updater = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(updater) + + +def entry(artifact_id, service_version, alias, alias_implementation=None): + return { + "alias": alias, + "aliasImplementation": alias_implementation or [], + "artifactId": artifact_id, + "serviceVersion": service_version, + } + + +AZURE_BLOB = entry("camel-test-infra-azure-storage-blob", "3.35.0", ["azure"], ["storage-blob"]) +AZURE_QUEUE = entry("camel-test-infra-azure-storage-queue", "3.35.0", ["azure"], ["storage-queue"]) +OLLAMA = entry("camel-test-infra-ollama", "0.31.2", ["ollama"]) +HIVEMQ = entry("camel-test-infra-hivemq", "2025.5", ["hive-mq"]) +HIVEMQ_SPARKPLUG = entry("camel-test-infra-hivemq", "camel", ["hive-mq"], ["sparkplug"]) +OBSERVABILITY = entry("camel-test-infra-observability", None, ["observability"]) + + +class PropertyPrefixTest(unittest.TestCase): + + def test_container_property_yields_normalized_prefix(self): + self.assertEqual("azure", updater.property_prefix("azure.container")) + self.assertEqual("hashicorpvault", updater.property_prefix("hashicorp.vault.container")) + self.assertEqual("kafka", updater.property_prefix("kafka.container.image")) + + def test_platform_specific_property_has_no_target(self): + # The Mojo skips these keys, so their version never reaches metadata.json + self.assertIsNone(updater.property_prefix("ollama.container.ppc64le")) + self.assertIsNone(updater.property_prefix("aws.container.s390x")) + self.assertIsNone(updater.property_prefix("tensorflow.serving.container.aarch64")) + self.assertIsNone(updater.property_prefix("artemis.container.amd64")) + + def test_version_metadata_property_has_no_target(self): + self.assertIsNone(updater.property_prefix("ollama.container.version.exclude")) + self.assertIsNone(updater.property_prefix("mongodb.container.version.include")) + self.assertIsNone(updater.property_prefix("milvus.container.version.freeze.major")) + self.assertIsNone(updater.property_prefix("rocketmq.container.image.version")) + + def test_non_container_property_has_no_target(self): + self.assertIsNone(updater.property_prefix("ollama.model")) + self.assertIsNone(updater.property_prefix("ollama.api.key")) + + +class MatchesTest(unittest.TestCase): + + def test_matches_service_alias(self): + self.assertTrue(updater.matches(OLLAMA, "ollama")) + self.assertFalse(updater.matches(OLLAMA, "milvus")) + + def test_matches_dashed_alias(self): + self.assertTrue(updater.matches(HIVEMQ, "hivemq")) + + def test_compound_prefix_matches_implementation_alias(self): + self.assertTrue(updater.matches(HIVEMQ_SPARKPLUG, "hivemqsparkplug")) + self.assertFalse(updater.matches(HIVEMQ, "hivemqsparkplug")) + + def test_matches_entry_of_another_module(self): + # azure.container lives in camel-test-infra-azure-common + self.assertTrue(updater.matches(AZURE_BLOB, "azure")) + self.assertTrue(updater.matches(AZURE_QUEUE, "azure")) + + +class UpdateMetadataTest(unittest.TestCase): + + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + + def write(self, entries): + path = os.path.join(self.tmp.name, "metadata.json") + with open(path, "w", encoding="utf-8") as f: + f.write(updater.jackson_format(entries)) + return path + + def read(self, path): + with open(path, "r", encoding="utf-8") as f: + return json.load(f) + + def update(self, prefix, old_version, new_version, path): + with contextlib.redirect_stdout(io.StringIO()): + return updater.update_metadata(prefix, old_version, new_version, path) + + def run_main(self): + out = io.StringIO() + with contextlib.redirect_stdout(out), contextlib.redirect_stderr(out): + updater.main() + return out.getvalue() + + def test_updates_every_entry_fed_by_a_shared_property(self): + path = self.write([AZURE_BLOB, AZURE_QUEUE, OLLAMA]) + + self.assertTrue(self.update("azure", "3.35.0", "3.36.0", path)) + + versions = {e["artifactId"]: e["serviceVersion"] for e in self.read(path)} + self.assertEqual("3.36.0", versions["camel-test-infra-azure-storage-blob"]) + self.assertEqual("3.36.0", versions["camel-test-infra-azure-storage-queue"]) + self.assertEqual("0.31.2", versions["camel-test-infra-ollama"]) + + def test_leaves_entries_on_another_version_alone(self): + path = self.write([AZURE_BLOB, AZURE_QUEUE]) + + self.assertFalse(self.update("azure", "3.30.0", "3.36.0", path)) + + self.assertEqual(["3.35.0", "3.35.0"], [e["serviceVersion"] for e in self.read(path)]) + + def test_version_separates_entries_sharing_a_service_alias(self): + path = self.write([HIVEMQ, HIVEMQ_SPARKPLUG]) + + self.assertTrue(self.update("hivemq", "2025.5", "2025.6", path)) + + self.assertEqual(["2025.6", "camel"], [e["serviceVersion"] for e in self.read(path)]) + + def test_missing_file_is_skipped(self): + self.assertFalse( + self.update("azure", "3.35.0", "3.36.0", os.path.join(self.tmp.name, "absent.json"))) + + def test_main_updates_all_metadata_files(self): + first = self.write([AZURE_BLOB, AZURE_QUEUE]) + second = os.path.join(self.tmp.name, "catalog.json") + with open(second, "w", encoding="utf-8") as f: + f.write(updater.jackson_format([AZURE_BLOB, AZURE_QUEUE])) + + with mock.patch("sys.argv", ["prog", "azure.container", "3.35.0", "3.36.0", first, second]): + self.run_main() + + for path in (first, second): + self.assertEqual(["3.36.0", "3.36.0"], [e["serviceVersion"] for e in self.read(path)]) + + def test_main_succeeds_when_the_property_has_no_metadata_target(self): + # CAMEL-24252: a bump with nothing to update must not abort the caller + path = self.write([OLLAMA]) + + with mock.patch("sys.argv", ["prog", "ollama.container.ppc64le", "v0.17.6", "v0.24.0", path]): + output = self.run_main() + + self.assertIn("nothing to update", output) + self.assertEqual("0.31.2", self.read(path)[0]["serviceVersion"]) + + def test_main_succeeds_when_no_entry_matches(self): + # CAMEL-24252: multi-container modules carry no serviceVersion + path = self.write([OBSERVABILITY]) + + with mock.patch("sys.argv", ["prog", "observability.perses.container", "v0.53.1", "v0.54.0", path]): + output = self.run_main() + + self.assertIn("nothing to update", output) + self.assertIsNone(self.read(path)[0]["serviceVersion"]) + + def test_main_rejects_missing_arguments(self): + with mock.patch("sys.argv", ["prog", "azure.container", "3.35.0"]): + with self.assertRaises(SystemExit) as raised: + self.run_main() + + self.assertEqual(1, raised.exception.code) + + +class JacksonFormatTest(unittest.TestCase): + + @unittest.skipUnless(METADATA_INFRA.is_file(), f"{METADATA_INFRA} not available") + def test_output_matches_the_generated_metadata(self): + # The formatter must reproduce the Maven build output byte for byte, + # otherwise every bump would rewrite the whole file + content = METADATA_INFRA.read_text(encoding="utf-8") + + self.assertEqual(content, updater.jackson_format(json.loads(content))) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/actions/check-container-upgrade/update-metadata-version.py b/.github/actions/check-container-upgrade/update-metadata-version.py index b4357169df2fa..cebcb629efa95 100644 --- a/.github/actions/check-container-upgrade/update-metadata-version.py +++ b/.github/actions/check-container-upgrade/update-metadata-version.py @@ -21,14 +21,35 @@ image version is bumped. Produces output matching Jackson's DefaultPrettyPrinter format so the result is identical to what the Maven build generates. +Entries are matched the same way CamelTestInfraGenerateMetadataMojo derives +serviceVersion: the property key prefix (the part before ".container") is +matched against the entry aliasImplementation/alias values. Matching on the +Maven module name does not work, because a container.properties file may be +shared across modules — azure.container lives in camel-test-infra-azure-common +while the metadata entries belong to camel-test-infra-azure-storage-blob and +camel-test-infra-azure-storage-queue. + +Some bumps legitimately have no metadata counterpart, and are reported as a +no-op rather than an error: + * platform-specific keys (.ppc64le/.s390x/.aarch64/.amd64), which the Mojo skips + * multi-container modules whose entries have no serviceVersion (observability) + * modules with no @InfraService entry at all (triton, tensorflow-serving, ...) + Usage: - python3 update-metadata-version.py [ ...] + python3 update-metadata-version.py [ ...] """ import json import os import sys +# Suffixes the metadata Mojo skips: platform-specific image variants never +# reach metadata.json, so a bump of those keys has no serviceVersion to update. +PLATFORM_SUFFIXES = (".ppc64le", ".s390x", ".aarch64", ".amd64") + +# Keys carrying version-check metadata rather than an image reference. +METADATA_MARKERS = (".version.exclude", ".version.include", ".version.freeze") + def jackson_format(data): """Serialize JSON to match Jackson's DefaultPrettyPrinter output.""" @@ -52,8 +73,51 @@ def jackson_format(data): return "\n".join(lines) -def update_metadata(artifact_id, old_version, new_version, metadata_file): - """Update serviceVersion for entries matching the artifact and old version.""" +def normalize(value): + """Normalize an alias or property prefix for matching, as the Mojo does.""" + return value.replace("-", "").replace(".", "").lower() + + +def property_prefix(property_name): + """ + Return the normalized prefix of a container property key, or None when the + key holds no image version the metadata could ever carry. + """ + key = property_name.lower() + + if key.endswith(PLATFORM_SUFFIXES) or key.endswith(".version"): + return None + if any(marker in key for marker in METADATA_MARKERS): + return None + if "container" not in key: + return None + + index = key.find(".container") + if index <= 0: + return None + + return normalize(key[:index]) + + +def matches(entry, prefix): + """ + Check whether the metadata entry is the one the property key feeds: the + prefix must equal, or end with, one of the entry aliases — the suffix match + covers compound keys such as hivemq.sparkplug.container. + """ + aliases = list(entry.get("aliasImplementation") or []) + aliases.extend(entry.get("alias") or []) + + for alias in aliases: + normalized = normalize(alias) + if normalized and prefix.endswith(normalized): + return True + + return False + + +def update_metadata(prefix, old_version, new_version, metadata_file): + """Update serviceVersion for entries fed by the bumped property.""" if not os.path.isfile(metadata_file): print(f"⚠️ {metadata_file} not found, skipping") return False @@ -63,10 +127,7 @@ def update_metadata(artifact_id, old_version, new_version, metadata_file): updated = False for entry in data: - if ( - entry.get("artifactId") == artifact_id - and entry.get("serviceVersion") == old_version - ): + if entry.get("serviceVersion") == old_version and matches(entry, prefix): entry["serviceVersion"] = new_version updated = True @@ -74,8 +135,6 @@ def update_metadata(artifact_id, old_version, new_version, metadata_file): with open(metadata_file, "w", encoding="utf-8") as f: f.write(jackson_format(data)) print(f"✅ Updated serviceVersion {old_version} → {new_version} in {metadata_file}") - else: - print(f"ℹ️ No matching entries for {artifact_id}/{old_version} in {metadata_file}") return updated @@ -83,27 +142,32 @@ def update_metadata(artifact_id, old_version, new_version, metadata_file): def main(): if len(sys.argv) < 5: print( - "Usage: update-metadata-version.py [ ...]" + "Usage: update-metadata-version.py [ ...]" ) sys.exit(1) - artifact_id = sys.argv[1] + property_name = sys.argv[1] old_version = sys.argv[2] new_version = sys.argv[3] metadata_files = sys.argv[4:] - failed_metadata_files = [] - for metadata_file in metadata_files: - if not update_metadata(artifact_id, old_version, new_version, metadata_file): - failed_metadata_files.append(metadata_file) - - if failed_metadata_files: - print( - "❌ No metadata target was updated in: " - + ", ".join(failed_metadata_files), - file=sys.stderr, - ) - sys.exit(1) + prefix = property_prefix(property_name) + if prefix is None: + print(f"ℹ️ {property_name} has no serviceVersion in the metadata, nothing to update") + return + + untouched = [ + metadata_file + for metadata_file in metadata_files + if not update_metadata(prefix, old_version, new_version, metadata_file) + ] + + if len(untouched) == len(metadata_files): + print(f"ℹ️ No metadata entry matches {property_name}/{old_version}, nothing to update") + elif untouched: + # The metadata files mirror each other, so an update landing in only + # some of them means one of them is out of sync. + print("⚠️ No matching entry in: " + ", ".join(untouched), file=sys.stderr) if __name__ == "__main__": diff --git a/.github/workflows/check-container-versions.yml b/.github/workflows/check-container-versions.yml index 9327a922bff8b..48e904709ccb7 100644 --- a/.github/workflows/check-container-versions.yml +++ b/.github/workflows/check-container-versions.yml @@ -306,12 +306,16 @@ jobs: # Create and switch to new branch git checkout -b "$BRANCH_NAME" - # Update serviceVersion in metadata.json files + # Update serviceVersion in metadata.json files. + # A failure here must not abort the loop: the remaining containers + # still deserve their own PR. METADATA_INFRA="test-infra/camel-test-infra-all/src/generated/resources/META-INF/metadata.json" METADATA_CATALOG="catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/test-infra/metadata.json" - python3 .github/actions/check-container-upgrade/update-metadata-version.py \ - "$MODULE_NAME" "$OLD_VERSION" "$NEW_VERSION" \ - "$METADATA_INFRA" "$METADATA_CATALOG" + if ! python3 .github/actions/check-container-upgrade/update-metadata-version.py \ + "$PROPERTY_NAME" "$OLD_VERSION" "$NEW_VERSION" \ + "$METADATA_INFRA" "$METADATA_CATALOG"; then + echo "⚠️ Failed to update the metadata for $PROPERTY_NAME, continuing" + fi # Add and commit changes git add "$FILE_PATH" "$METADATA_INFRA" "$METADATA_CATALOG" diff --git a/.github/workflows/pr-ci-scripts-validation.yml b/.github/workflows/pr-ci-scripts-validation.yml new file mode 100644 index 0000000000000..9697d7c676f21 --- /dev/null +++ b/.github/workflows/pr-ci-scripts-validation.yml @@ -0,0 +1,47 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: PR CI scripts validation +on: + pull_request: + branches: + - main + paths: + - '.github/actions/check-container-upgrade/**' + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + test: + if: github.repository == 'apache/camel' + runs-on: ubuntu-latest + steps: + - name: Checkout camel repo + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Test the container upgrade scripts + shell: bash + run: | + cd .github/actions/check-container-upgrade + python3 -m unittest discover --verbose diff --git a/.gitignore b/.gitignore index 9a9a12cf53b67..e03f8a64b89de 100644 --- a/.gitignore +++ b/.gitignore @@ -27,6 +27,7 @@ components/camel-cxf/activemq-data .flattened-pom.xml .java-version node_modules/ +__pycache__/ mvnd.zip* .camel-jbang .camel-jbang-run