Skip to content

Commit d16fc4c

Browse files
sawenzelclaude
sawenzel
and
claude
committed
Add a single entrypoint for the O2DPG tests
This adds test/run_tests.sh and fixes two exit-code problems in the workflow tests. - The generator, workflow and RelVal sub-tests are now selected and aggregated in one script instead of in the alidist recipe body. - run_workflow_tests.sh dropped ret_global_pwg from its final exit code, so a failed PWG workflow creation reported FAILED and still passed. It is now included. - The AnalysisQC execution block was gated on changed_analysis_qc alone while announcing itself for MC/bin changes too. Both now agree. - test/tests/run_tests_selection.sh checks selection and aggregation offline. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
1 parent 41b8e47 commit d16fc4c

4 files changed

Lines changed: 194 additions & 4 deletions

File tree

‎test/README.md‎

Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ At the moment, the tests focus on generator configurations and custom generators
77

88
Tests are run via
99
```bash
10-
${O2DPG_ROOT}/test/run_tests.sh [--fail-immediately]
10+
${O2DPG_ROOT}/test/run_tests.sh [--fail-immediately] [--keep-artifacts] [SUBTEST...]
1111
```
1212

1313
Tests are run for changed
@@ -56,7 +56,23 @@ O2DPG_TEST_REPO_DIR=</path/to/source/O2DPG> ${O2DPG_ROOT}/test/run_tests.sh [--f
5656
```
5757
If you are inside the source directory, you can simply run
5858
```bash
59-
${O2DPG_ROOT}/test/run_tests.sh [--fail-immediately]
59+
${O2DPG_ROOT}/test/run_tests.sh [--fail-immediately] [--keep-artifacts] [SUBTEST...]
60+
```
61+
62+
### Running a subset
63+
64+
`run_tests.sh` runs the generator, workflow and RelVal sub-tests. To run only
65+
some of them, name them:
66+
67+
```bash
68+
${O2DPG_ROOT}/test/run_tests.sh generator relval
69+
```
70+
71+
The offline harnesses under `test/tests/` check the entrypoint's selection and
72+
exit-code aggregation without needing an O2 environment:
73+
74+
```bash
75+
bash test/tests/run_tests_selection.sh
6076
```
6177

6278
### Keeping all test artifacts

‎test/run_tests.sh‎

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
#!/bin/bash
2+
3+
# Single entrypoint for the O2DPG tests. Runs the generator, workflow and
4+
# RelVal sub-tests, aggregates their exit codes and prints the collected log.
5+
6+
set -u
7+
8+
SUBTEST_DIR=${O2DPG_TEST_SUBTEST_DIR:-$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)}
9+
10+
ALL_SUBTESTS="generator workflow relval"
11+
12+
print_usage()
13+
{
14+
echo
15+
echo "usage: run_tests.sh [--fail-immediately] [--keep-artifacts] [SUBTEST...]"
16+
echo
17+
echo " SUBTEST : one or more of: ${ALL_SUBTESTS} (default: all)"
18+
echo
19+
echo " FLAGS:"
20+
echo
21+
echo " --fail-immediately : stop after the first failing sub-test"
22+
echo " --keep-artifacts : keep simulation artifacts, not just the logs"
23+
echo
24+
echo " ENVIRONMENT VARIABLES:"
25+
echo
26+
echo " O2DPG_TEST_REPO_DIR : the source repository to test"
27+
echo " O2DPG_TEST_HASH_BASE : base hash for the changed-file diff (optional)"
28+
echo " O2DPG_TEST_HASH_HEAD : head hash for the changed-file diff (optional)"
29+
echo
30+
}
31+
32+
fail_immediately=
33+
subtest_args=()
34+
selected=()
35+
36+
while [ "$#" -gt 0 ] ; do
37+
case $1 in
38+
--fail-immediately ) fail_immediately=1 ; subtest_args+=("$1") ; shift ;;
39+
--keep-artifacts ) subtest_args+=("$1") ; shift ;;
40+
--help|-h ) print_usage ; exit 1 ;;
41+
generator|workflow|relval ) selected+=("$1") ; shift ;;
42+
* ) echo "Unknown argument ${1}" ; print_usage ; exit 1 ;;
43+
esac
44+
done
45+
46+
[[ "${#selected[@]}" == "0" ]] && read -r -a selected <<< "${ALL_SUBTESTS}"
47+
48+
# Geant4 dataset variables, if a Geant4 installation is in the environment.
49+
if [[ -n "${G4INSTALL:-}" ]] ; then
50+
eval "$("${G4INSTALL}/bin/geant4-config" --datasets |
51+
sed -e 's/[^ ]* //' -e 's/G4/export G4/' -e 's/DATA /DATA=/')"
52+
fi
53+
54+
# LHAPDF data, needed by several generator configurations.
55+
if [[ -z "${LHAPDF_DATA_PATH:-}" && -n "${LHAPDF_ROOT:-}" ]] ; then
56+
export LHAPDF_DATA_PATH="${LHAPDF_ROOT}/share/LHAPDF:${LHAPDF_PDFSETS_ROOT:-}/share/LHAPDF"
57+
echo "Set LHAPDF_DATA_PATH to ${LHAPDF_DATA_PATH}"
58+
fi
59+
60+
ret_global=0
61+
for subtest in "${selected[@]}" ; do
62+
script="${SUBTEST_DIR}/run_${subtest}_tests.sh"
63+
if [[ ! -x "${script}" ]] ; then
64+
echo "ERROR: sub-test script ${script} not found or not executable"
65+
ret_global=1
66+
[[ "${fail_immediately}" == "1" ]] && break
67+
continue
68+
fi
69+
echo
70+
echo "==> START SUBTEST: ${subtest} <=="
71+
forward=()
72+
for arg in "${subtest_args[@]:-}" ; do
73+
[[ -z "${arg}" ]] && continue
74+
grep -q -- "${arg}" "${script}" && forward+=("${arg}")
75+
done
76+
"${script}" "${forward[@]}"
77+
ret_this=$?
78+
echo "==> END SUBTEST: ${subtest} (exit ${ret_this}) <=="
79+
if [[ "${ret_this}" != "0" ]] ; then
80+
ret_global=${ret_this}
81+
[[ "${fail_immediately}" == "1" ]] && break
82+
fi
83+
done
84+
85+
echo
86+
if [[ "${ret_global}" != "0" ]] ; then
87+
echo "error detected in O2DPG tests, see above"
88+
else
89+
echo "O2DPG tests passed"
90+
fi
91+
92+
exit ${ret_global}

‎test/run_workflow_tests.sh‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -293,7 +293,7 @@ fi
293293
# prepare our local test directory for bin tests
294294
# global return code for PWGs
295295
ret_analysis_qc=0
296-
if [[ "${changed_analysis_qc}" != "" ]] ; then
296+
if [[ "${changed_analysis_qc}" != "" || "${changed_sim_bin}" != "" ]] ; then
297297
rm -rf ${TEST_PARENT_DIR_BIN} 2>/dev/null
298298
mkdir -p ${TEST_PARENT_DIR_BIN} 2>/dev/null
299299
pushd ${TEST_PARENT_DIR_BIN} > /dev/null
@@ -336,7 +336,7 @@ if [[ "${changed_anchored}" != "" ]] ; then
336336
popd > /dev/null
337337
fi
338338

339-
RET=$(( ret_analysis_qc + ret_global_anchored ))
339+
RET=$(( ret_global_pwg + ret_analysis_qc + ret_global_anchored ))
340340

341341
echo
342342
[[ "${RET}" != "0" ]] && echo_red "There were errors, please check!" || echo_green "All required workflow tests successful"

‎test/tests/run_tests_selection.sh‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
#!/bin/bash
2+
# Offline harness for run_tests.sh: stubs the sub-test scripts and checks that
3+
# selection and exit-code aggregation behave.
4+
5+
set -u
6+
HERE=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
7+
ENTRY="${HERE}/../run_tests.sh"
8+
FAILURES=0
9+
10+
check()
11+
{
12+
local label=$1 expected=$2 actual=$3
13+
if [[ "${expected}" == "${actual}" ]] ; then
14+
echo "ok - ${label}"
15+
else
16+
echo "FAIL - ${label}: expected '${expected}', got '${actual}'"
17+
FAILURES=$((FAILURES + 1))
18+
fi
19+
}
20+
21+
# A stub directory with three scripts whose exit codes we control via env vars.
22+
make_stubs()
23+
{
24+
local dir=$1
25+
mkdir -p "${dir}"
26+
for name in generator workflow relval ; do
27+
cat > "${dir}/run_${name}_tests.sh" <<EOF
28+
#!/bin/bash
29+
echo "STUB ${name} ran"
30+
exit \${STUB_${name^^}_RC:-0}
31+
EOF
32+
chmod +x "${dir}/run_${name}_tests.sh"
33+
done
34+
}
35+
36+
run_entry()
37+
{
38+
# usage: run_entry <stubdir> [args...]; prints nothing, sets OUT and RC
39+
local dir=$1 ; shift
40+
OUT=$(O2DPG_TEST_SUBTEST_DIR="${dir}" O2DPG_ROOT="${HERE}/../.." \
41+
O2DPG_TEST_REPO_DIR="${HERE}/../.." \
42+
bash "${ENTRY}" "$@" 2>&1)
43+
RC=$?
44+
}
45+
46+
TMP=$(mktemp -d)
47+
make_stubs "${TMP}/subtests"
48+
49+
# 1. all three sub-tests run by default and a clean run exits 0
50+
run_entry "${TMP}/subtests"
51+
check "default selection exits 0" 0 "${RC}"
52+
check "default runs generator" 1 "$(grep -c 'STUB generator ran' <<< "${OUT}")"
53+
check "default runs workflow" 1 "$(grep -c 'STUB workflow ran' <<< "${OUT}")"
54+
check "default runs relval" 1 "$(grep -c 'STUB relval ran' <<< "${OUT}")"
55+
56+
# 2. an explicit selection runs only what was asked for
57+
run_entry "${TMP}/subtests" relval
58+
check "explicit selection exits 0" 0 "${RC}"
59+
check "explicit selection skips generator" 0 "$(grep -c 'STUB generator ran' <<< "${OUT}")"
60+
check "explicit selection runs relval" 1 "$(grep -c 'STUB relval ran' <<< "${OUT}")"
61+
62+
# 3. a failing sub-test makes the entrypoint fail, and the others still run
63+
STUB_GENERATOR_RC=3 run_entry "${TMP}/subtests"
64+
check "failing generator propagates" 3 "${RC}"
65+
check "failure does not stop workflow" 1 "$(grep -c 'STUB workflow ran' <<< "${OUT}")"
66+
67+
# 4. --fail-immediately stops after the first failure
68+
STUB_GENERATOR_RC=3 run_entry "${TMP}/subtests" --fail-immediately
69+
check "fail-immediately propagates" 3 "${RC}"
70+
check "fail-immediately stops early" 0 "$(grep -c 'STUB relval ran' <<< "${OUT}")"
71+
72+
# 5. a later failure is not masked by an earlier success
73+
STUB_RELVAL_RC=4 run_entry "${TMP}/subtests"
74+
check "late failure propagates" 4 "${RC}"
75+
76+
echo
77+
if [[ "${FAILURES}" == "0" ]] ; then
78+
echo "All run_tests.sh selection tests passed"
79+
else
80+
echo "${FAILURES} test(s) failed"
81+
fi
82+
exit $(( FAILURES > 0 ? 1 : 0 ))

0 commit comments

Comments
 (0)