Skip to content

Isolate app config test fixtures #218

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
yoshifuminakamura merged 1 commit into from
Sep 17, 2026
Merged

Isolate app config test fixtures #218

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
54 changes: 38 additions & 16 deletions scripts/tests/test_matrix_generate_filters.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,37 +7,59 @@ REPO_DIR=$(cd "${SCRIPT_DIR}/../.." && pwd)
TMP_DIR=$(mktemp -d)
trap 'rm -rf "${TMP_DIR}"' EXIT

mkdir -p "${TMP_DIR}/project"
ln -s "${REPO_DIR}/config" "${TMP_DIR}/project/config"
ln -s "${REPO_DIR}/programs" "${TMP_DIR}/project/programs"
mkdir -p "${TMP_DIR}/project/config" "${TMP_DIR}/project/programs/filterapp"
ln -s "${REPO_DIR}/scripts" "${TMP_DIR}/project/scripts"

cat > "${TMP_DIR}/project/config/system.csv" <<'EOF'
system,mode,tag_build,tag_run,queue,queue_group
FilterSystem,cross,build-tag,run-tag,SLURM_TEST,debug
EOF

cat > "${TMP_DIR}/project/config/queue.csv" <<'EOF'
queue,submit_cmd,template
SLURM_TEST,sbatch,"-p ${queue_group} -t ${elapse} -N ${nodes} --ntasks-per-node=${numproc_node} --cpus-per-task=${nthreads}"
EOF

cat > "${TMP_DIR}/project/config/system_info.csv" <<'EOF'
system,name,cpu_name,cpu_per_node,cpu_cores,gpu_name,gpu_per_node,memory,display_order
FilterSystem,FilterSystem,CPU,1,32,GPU,1,64 GB,1
EOF

cat > "${TMP_DIR}/project/programs/filterapp/list.csv" <<'EOF'
system,enable,nodes,numproc_node,nthreads,elapse
FilterSystem,yes,1,2,3,0:10:00
FilterSystem,yes,2,2,3,0:10:00
FilterSystem,yes,4,2,3,0:10:00
EOF
printf '#!/bin/bash\n' > "${TMP_DIR}/project/programs/filterapp/build.sh"
printf '#!/bin/bash\n' > "${TMP_DIR}/project/programs/filterapp/run.sh"

pushd "${TMP_DIR}/project" >/dev/null

bash scripts/matrix_generate.sh code=sbd system=RIKYU nodes=1
grep -q '^sbd_RIKYU_N1_P4_T32_run:' .gitlab-ci.generated.yml
bash scripts/matrix_generate.sh code=filterapp system=FilterSystem nodes=1
grep -q '^filterapp_FilterSystem_N1_P2_T3_run:' .gitlab-ci.generated.yml
grep -q 'name: $CI_COMMIT_BRANCH' .gitlab-ci.generated.yml
if grep -q '^sbd_RIKYU_N2_P4_T32_run:' .gitlab-ci.generated.yml; then
echo "nodes=1 filter must not emit the SBD RIKYU 2-node job" >&2
if grep -q '^filterapp_FilterSystem_N2_P2_T3_run:' .gitlab-ci.generated.yml; then
echo "nodes=1 filter must not emit the 2-node fixture job" >&2
exit 1
fi
if grep -q '^sbd_RIKYU_N4_P4_T32_run:' .gitlab-ci.generated.yml; then
echo "nodes=1 filter must not emit the SBD RIKYU 4-node job" >&2
if grep -q '^filterapp_FilterSystem_N4_P2_T3_run:' .gitlab-ci.generated.yml; then
echo "nodes=1 filter must not emit the 4-node fixture job" >&2
exit 1
fi

bash scripts/matrix_generate.sh code=sbd system=RIKYU nodes='2, 4'
if grep -q '^sbd_RIKYU_N1_P4_T32_run:' .gitlab-ci.generated.yml; then
echo "nodes=2,4 filter must not emit the SBD RIKYU 1-node job" >&2
bash scripts/matrix_generate.sh code=filterapp system=FilterSystem nodes='2, 4'
if grep -q '^filterapp_FilterSystem_N1_P2_T3_run:' .gitlab-ci.generated.yml; then
echo "nodes=2,4 filter must not emit the 1-node fixture job" >&2
exit 1
fi
grep -q '^sbd_RIKYU_N2_P4_T32_run:' .gitlab-ci.generated.yml
grep -q '^sbd_RIKYU_N4_P4_T32_run:' .gitlab-ci.generated.yml
grep -q '^filterapp_FilterSystem_N2_P2_T3_run:' .gitlab-ci.generated.yml
grep -q '^filterapp_FilterSystem_N4_P2_T3_run:' .gitlab-ci.generated.yml

BK_GITLAB_ENVIRONMENT=develop bash scripts/matrix_generate.sh code=sbd system=RIKYU nodes=1
BK_GITLAB_ENVIRONMENT=develop bash scripts/matrix_generate.sh code=filterapp system=FilterSystem nodes=1
grep -q 'name: "develop"' .gitlab-ci.generated.yml

if BK_GITLAB_ENVIRONMENT='bad scope' bash scripts/matrix_generate.sh code=sbd system=RIKYU nodes=1 >/dev/null 2>&1; then
if BK_GITLAB_ENVIRONMENT='bad scope' bash scripts/matrix_generate.sh code=filterapp system=FilterSystem nodes=1 >/dev/null 2>&1; then
echo "invalid BK_GITLAB_ENVIRONMENT must fail matrix generation" >&2
exit 1
fi
Expand Down
37 changes: 32 additions & 5 deletions scripts/tests/test_scheduler_extra_args.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,9 @@ test "$(get_scheduler_extra_args RIKYU)" = "--account=global"
test "$(get_scheduler_extra_args Fugaku)" = "--account=global"

tmpdir=""
project_dir=""
estimate_tmpdir=$(mktemp -d)
trap 'rm -rf "$tmpdir" "$estimate_tmpdir"; rm -f "${REPO_DIR}/script.sh"' EXIT
trap 'rm -rf "$tmpdir" "$estimate_tmpdir"' EXIT

mkdir -p "$estimate_tmpdir/app"
if has_estimate_script "$estimate_tmpdir/app"; then
Expand All @@ -50,6 +51,30 @@ if has_estimate_script "$estimate_tmpdir/app"; then
fi

tmpdir=$(mktemp -d)
project_dir="$tmpdir/project"

mkdir -p "$project_dir/config" "$project_dir/programs/schedulerapp"
ln -s "${REPO_DIR}/scripts" "$project_dir/scripts"

cat > "$project_dir/config/system.csv" <<'EOF'
system,mode,tag_build,tag_run,queue,queue_group
RC_GH200,native,,,SLURM_RC,gh200
Fugaku,cross,,,FJ,small
EOF

cat > "$project_dir/config/system_info.csv" <<'EOF'
system,name,cpu_name,cpu_per_node,cpu_cores,gpu_name,gpu_per_node,memory,display_order
RC_GH200,RC_GH200,CPU,1,72,GPU,1,120 GB,1
Fugaku,Fugaku,CPU,1,48,-,-,32 GB,2
EOF

cat > "$project_dir/programs/schedulerapp/list.csv" <<'EOF'
system,enable,nodes,numproc_node,nthreads,elapse
RC_GH200,yes,1,1,1,0:05:00
Fugaku,yes,1,2,3,0:05:00
EOF
printf '#!/bin/bash\n' > "$project_dir/programs/schedulerapp/build.sh"
printf '#!/bin/bash\n' > "$project_dir/programs/schedulerapp/run.sh"

cat >"$tmpdir/sbatch" <<'SCRIPT'
#!/bin/bash
Expand All @@ -65,14 +90,15 @@ unset BK_SCHEDULER_EXTRA_ARGS_RIKYU
unset BK_SCHEDULER_EXTRA_ARGS_Fugaku
unset BK_SCHEDULER_EXTRA_ARGS_RC_GH200

bash scripts/test_submit_build.sh qws 5 >/dev/null
pushd "$project_dir" >/dev/null
bash scripts/test_submit_build.sh schedulerapp 1 >/dev/null
if grep -q -- "--account=rkp00010" "$BK_TEST_SBATCH_ARGS_FILE"; then
echo "RC_GH200 must not derive --account from BK_ALLOCATION_PROJECT_ID" >&2
exit 1
fi

export BK_SCHEDULER_EXTRA_ARGS_RC_GH200="--account=explicit-rc"
bash scripts/test_submit_build.sh qws 5 >/dev/null
bash scripts/test_submit_build.sh schedulerapp 1 >/dev/null
grep -q -- "--account=explicit-rc" "$BK_TEST_SBATCH_ARGS_FILE"

cat >"$tmpdir/pjsub" <<'SCRIPT'
Expand All @@ -86,15 +112,16 @@ export BK_ALLOCATION_PROJECT_ID="ra000009"
unset BK_SCHEDULER_EXTRA_ARGS
unset BK_SCHEDULER_EXTRA_ARGS_Fugaku

bash scripts/test_submit.sh qws 2 >/dev/null
bash scripts/test_submit.sh schedulerapp 2 >/dev/null
grep -q -- "-g ra000009" "$BK_TEST_PJSUB_ARGS_FILE"

unset BK_ALLOCATION_PROJECT_ID
bash scripts/test_submit.sh qws 2 >/dev/null
bash scripts/test_submit.sh schedulerapp 2 >/dev/null
if grep -q -- "-g" "$BK_TEST_PJSUB_ARGS_FILE"; then
echo "Fugaku must not pass -g when BK_ALLOCATION_PROJECT_ID is unset" >&2
exit 1
fi
popd >/dev/null

popd >/dev/null

Expand Down
Loading