From bd4362f05ede10b589a8acfa20ca9a4bd58a834d Mon Sep 17 00:00:00 2001 From: Bikash Shaw Date: Wed, 3 Jun 2026 16:16:02 +0530 Subject: [PATCH 1/3] OCPBUGS-84961: apply full test processing pipeline to external binary tests External binary tests loaded via TestBinaries.ListTests() were missing the processing pipeline that origin built-in tests go through in InitializeOpenShiftTestsExtensionFramework(). This caused external binary tests to lack [Suite:openshift/conformance/...] tags, dropping the openshift/conformance suite from ~4000 to ~190 tests on OCP 4.21+. Apply the same processing steps to external binary test specs: - filterOutDisabledSpecs(): removes tests known to be broken - addEnvironmentSelectors(): adds [Skipped:] markers for platform/ network/topology-specific tests - addLabelsToSpecs(): adds labels like [Serial] where needed - appendSuiteNames(): adds [Suite:openshift/conformance/...] tags All four functions have guards to skip tests that already have the relevant annotations, so they are safe to call on specs that may already be partially processed by their source binary. Tested on OCP 4.21.8 GA cluster: - Before fix: 184 tests in openshift/conformance suite - After fix: 3895 tests in openshift/conformance suite Co-Authored-By: Claude Opus 4.6 --- pkg/test/extensions/binary.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/pkg/test/extensions/binary.go b/pkg/test/extensions/binary.go index b5e5469e8604..c7d716349992 100644 --- a/pkg/test/extensions/binary.go +++ b/pkg/test/extensions/binary.go @@ -778,6 +778,15 @@ func (binaries TestBinaries) ListTests(ctx context.Context, parallelism int, env return nil, fmt.Errorf("encountered errors while listing tests: %s", strings.Join(errs, ";")) } + var baseSpecs extensiontests.ExtensionTestSpecs + for _, spec := range allTests { + baseSpecs = append(baseSpecs, spec.ExtensionTestSpec) + } + baseSpecs = filterOutDisabledSpecs(baseSpecs) + addEnvironmentSelectors(baseSpecs) + addLabelsToSpecs(baseSpecs) + appendSuiteNames(baseSpecs) + return allTests, nil } From 25317e49326cc3a5e4749d8671259a834bacb1d5 Mon Sep 17 00:00:00 2001 From: Bikash Shaw Date: Wed, 3 Jun 2026 22:27:42 +0530 Subject: [PATCH 2/3] OCPBUGS-84961: preserve original test name for external binary dispatch appendSuiteNames() mutates spec.Name by appending [Suite:] suffixes. External binaries register tests without these suffixes, so dispatching the modified name causes "no such tests" errors. Fix by saving OriginalName before mutation and using it (via rawName) when dispatching tests to external binaries via RunTests(). Co-Authored-By: Claude Opus 4.6 (1M context) --- pkg/test/extensions/binary.go | 9 +++++++++ pkg/test/ginkgo/test_runner.go | 6 +++++- pkg/test/ginkgo/test_suite.go | 6 +++++- 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/pkg/test/extensions/binary.go b/pkg/test/extensions/binary.go index c7d716349992..91fff2c0c61e 100644 --- a/pkg/test/extensions/binary.go +++ b/pkg/test/extensions/binary.go @@ -785,6 +785,15 @@ func (binaries TestBinaries) ListTests(ctx context.Context, parallelism int, env baseSpecs = filterOutDisabledSpecs(baseSpecs) addEnvironmentSelectors(baseSpecs) addLabelsToSpecs(baseSpecs) + + // Preserve original test names before appendSuiteNames mutates them. + // External binaries register tests without [Suite:] suffixes, so dispatch + // must use the original name. + for _, spec := range baseSpecs { + if spec.OriginalName == "" { + spec.OriginalName = spec.Name + } + } appendSuiteNames(baseSpecs) return allTests, nil diff --git a/pkg/test/ginkgo/test_runner.go b/pkg/test/ginkgo/test_runner.go index f367d405d0e1..b2296b6d90f4 100644 --- a/pkg/test/ginkgo/test_runner.go +++ b/pkg/test/ginkgo/test_runner.go @@ -337,7 +337,11 @@ func (c *commandContext) RunTestInNewProcess(ctx context.Context, test *testCase timeout = test.testTimeout } - results := test.binary.RunTests(ctx, timeout, testEnv, test.name) + dispatchName := test.rawName + if dispatchName == "" { + dispatchName = test.name + } + results := test.binary.RunTests(ctx, timeout, testEnv, dispatchName) if len(results) != 1 { fmt.Fprintf(os.Stderr, "warning: expected 1 result from external binary; received %d", len(results)) } diff --git a/pkg/test/ginkgo/test_suite.go b/pkg/test/ginkgo/test_suite.go index 67d54e7b1acb..1ac1a13f49c4 100644 --- a/pkg/test/ginkgo/test_suite.go +++ b/pkg/test/ginkgo/test_suite.go @@ -16,9 +16,13 @@ var re = regexp.MustCompile(`.*\[Timeout:(.[^\]]*)\]`) func extensionTestSpecsToOriginTestCases(specs extensions.ExtensionTestSpecs) ([]*testCase, error) { var tests []*testCase for _, spec := range specs { + rawName := spec.Name + if spec.OriginalName != "" { + rawName = spec.OriginalName + } tc := &testCase{ name: spec.Name, - rawName: spec.Name, + rawName: rawName, binary: spec.Binary, spec: spec, } From 167e2267db4d1677717eeafcbde597f6d33dccaa Mon Sep 17 00:00:00 2001 From: Bikash Shaw Date: Thu, 4 Jun 2026 13:53:04 +0530 Subject: [PATCH 3/3] OCPBUGS-84961: save OriginalName before all origin-side mutations Move OriginalName assignment before filterOutDisabledSpecs, addEnvironmentSelectors, addLabelsToSpecs, and appendSuiteNames. Always overwrite OriginalName since some binaries set it for history tracking which is not suitable for dispatch. Co-Authored-By: Claude Opus 4.6 (1M context) --- pkg/test/extensions/binary.go | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/pkg/test/extensions/binary.go b/pkg/test/extensions/binary.go index 91fff2c0c61e..38b71d882b56 100644 --- a/pkg/test/extensions/binary.go +++ b/pkg/test/extensions/binary.go @@ -782,18 +782,16 @@ func (binaries TestBinaries) ListTests(ctx context.Context, parallelism int, env for _, spec := range allTests { baseSpecs = append(baseSpecs, spec.ExtensionTestSpec) } - baseSpecs = filterOutDisabledSpecs(baseSpecs) - addEnvironmentSelectors(baseSpecs) - addLabelsToSpecs(baseSpecs) - // Preserve original test names before appendSuiteNames mutates them. - // External binaries register tests without [Suite:] suffixes, so dispatch - // must use the original name. + // Save current binary-registered name before any origin-side mutations. + // External binaries only recognize the name they registered, not names + // modified by addLabelsToSpecs/addEnvironmentSelectors/appendSuiteNames. for _, spec := range baseSpecs { - if spec.OriginalName == "" { - spec.OriginalName = spec.Name - } + spec.OriginalName = spec.Name } + baseSpecs = filterOutDisabledSpecs(baseSpecs) + addEnvironmentSelectors(baseSpecs) + addLabelsToSpecs(baseSpecs) appendSuiteNames(baseSpecs) return allTests, nil