From 420aff82c4acaf232128a5d32361f9509de2ad1f Mon Sep 17 00:00:00 2001 From: JoshuaSBrown Date: Tue, 24 Mar 2026 13:54:43 -0400 Subject: [PATCH 1/9] feature: add core unit testing to CI --- .gitlab/stage_unit.yml | 29 +++++++++++++++++++++++++++++ core/docker/Dockerfile | 11 ++++++++++- 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/.gitlab/stage_unit.yml b/.gitlab/stage_unit.yml index 70e4e5a98..e1536ba73 100644 --- a/.gitlab/stage_unit.yml +++ b/.gitlab/stage_unit.yml @@ -1,6 +1,7 @@ --- include: - local: .gitlab/stage_build.yml + - local: .gitlab/common.yml run-ws-unit-job: # Either the web container needs to be rebuilt or the container needs to be @@ -126,3 +127,31 @@ run-authz-unit-job: - echo "-c 'cd /datafed/source; /opt/datafed/dependencies/bin/cmake --build build --target test'" >> run_globus.sh - chmod +x run_globus.sh - ./run_globus.sh + + +run-core-unit-job: + needs: ["run-core-build-job"] + stage: unit + variables: + PROJECT: "datafed" + COMPONENT: "core" + GIT_STRATEGY: clone + INTERMEDIATE_LAYER_NAME: "build" + tags: + - ci-datafed-core + - docker + script: + - BRANCH_LOWER=$(echo "$CI_COMMIT_REF_NAME" | tr '[:upper:]' '[:lower:]') + - docker login "${REGISTRY}" -u "${HARBOR_USER}" -p "${HARBOR_DATAFED_GITLAB_CI_REGISTRY_TOKEN}" + - ./scripts/container_stop.sh -n "core-unit-" -p + - random_string=$(bash -c "cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 10 | head -n 1") + - | + docker run --rm \ + --name "core-unit-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}" \ + --entrypoint bash \ + -t "${REGISTRY}/${PROJECT}/${COMPONENT}-${INTERMEDIATE_LAYER_NAME}-${BRANCH_LOWER}:${CI_COMMIT_SHA}" \ + -c 'cd /datafed/source && \ + /opt/datafed/dependencies/bin/ctest \ + --test-dir build \ + -LE "integration|fixture" \ + --output-on-failure' diff --git a/core/docker/Dockerfile b/core/docker/Dockerfile index b1981e9ba..dde9e0e7d 100644 --- a/core/docker/Dockerfile +++ b/core/docker/Dockerfile @@ -43,6 +43,11 @@ COPY ./cmake ${BUILD_DIR}/cmake COPY ./core/docker/entrypoint.sh ${BUILD_DIR}/core/docker/ COPY ./core/server ${BUILD_DIR}/core/server +# --------------------------------------------------------------------------- +# CHANGED: Enable unit and integration tests so the intermediate image +# contains all test binaries. The production 'core' stage below still only +# copies the server binary, so tests never land in the production image. +# --------------------------------------------------------------------------- RUN ${DATAFED_DEPENDENCIES_ROOT}/scripts/generate_dependencies_config.sh && \ ${BUILD_DIR}/scripts/generate_datafed.sh && \ ${DATAFED_DEPENDENCIES_INSTALL_PATH}/bin/cmake -S. -B build \ @@ -53,10 +58,14 @@ RUN ${DATAFED_DEPENDENCIES_ROOT}/scripts/generate_dependencies_config.sh && \ -DBUILD_DOCS=False \ -DBUILD_PYTHON_CLIENT=False \ -DBUILD_FOXX=False \ - -DENABLE_INTEGRATION_TESTS=False + -DENABLE_UNIT_TESTS=True \ + -DENABLE_INTEGRATION_TESTS=True RUN ${DATAFED_DEPENDENCIES_INSTALL_PATH}/bin/cmake --build build -j 8 RUN ${DATAFED_DEPENDENCIES_INSTALL_PATH}/bin/cmake --build build --target install +# =========================================================================== +# Production image — unchanged. Only the server binary is copied in. +# =========================================================================== FROM ${RUNTIME} AS core SHELL ["/bin/bash", "-c"] From 0cead873ed1b4efd169837f0867d9b93b2c867e5 Mon Sep 17 00:00:00 2001 From: JoshuaSBrown Date: Tue, 24 Mar 2026 16:51:29 -0400 Subject: [PATCH 2/9] refactor: leave off integration tests for first step. --- core/docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/docker/Dockerfile b/core/docker/Dockerfile index dde9e0e7d..eb1ebc36e 100644 --- a/core/docker/Dockerfile +++ b/core/docker/Dockerfile @@ -59,7 +59,7 @@ RUN ${DATAFED_DEPENDENCIES_ROOT}/scripts/generate_dependencies_config.sh && \ -DBUILD_PYTHON_CLIENT=False \ -DBUILD_FOXX=False \ -DENABLE_UNIT_TESTS=True \ - -DENABLE_INTEGRATION_TESTS=True + -DENABLE_INTEGRATION_TESTS=False RUN ${DATAFED_DEPENDENCIES_INSTALL_PATH}/bin/cmake --build build -j 8 RUN ${DATAFED_DEPENDENCIES_INSTALL_PATH}/bin/cmake --build build --target install From 3096a7a5ee39b228c17307bef022d900b491b7d2 Mon Sep 17 00:00:00 2001 From: Joshua S Brown Date: Wed, 25 Mar 2026 06:20:23 -0400 Subject: [PATCH 3/9] Apply suggestions from code review Co-authored-by: sourcery-ai[bot] <58596630+sourcery-ai[bot]@users.noreply.github.com> --- .gitlab/stage_unit.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/stage_unit.yml b/.gitlab/stage_unit.yml index e1536ba73..8a246e63e 100644 --- a/.gitlab/stage_unit.yml +++ b/.gitlab/stage_unit.yml @@ -142,7 +142,7 @@ run-core-unit-job: - docker script: - BRANCH_LOWER=$(echo "$CI_COMMIT_REF_NAME" | tr '[:upper:]' '[:lower:]') - - docker login "${REGISTRY}" -u "${HARBOR_USER}" -p "${HARBOR_DATAFED_GITLAB_CI_REGISTRY_TOKEN}" + - echo "${HARBOR_DATAFED_GITLAB_CI_REGISTRY_TOKEN}" | docker login "${REGISTRY}" -u "${HARBOR_USER}" --password-stdin - ./scripts/container_stop.sh -n "core-unit-" -p - random_string=$(bash -c "cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 10 | head -n 1") - | From 6be23f4200bb1c1da3c13b741522cfc896fdddb5 Mon Sep 17 00:00:00 2001 From: JoshuaSBrown Date: Wed, 25 Mar 2026 06:26:01 -0400 Subject: [PATCH 4/9] fix: make sure build container in intermediate stage exists. --- .gitlab/build/force_build_core_image.yml | 4 +++- .gitlab/build/retag_core_image.yml | 4 +++- .gitlab/stage_unit.yml | 1 - 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.gitlab/build/force_build_core_image.yml b/.gitlab/build/force_build_core_image.yml index 1520cd505..cd0ad710b 100644 --- a/.gitlab/build/force_build_core_image.yml +++ b/.gitlab/build/force_build_core_image.yml @@ -14,7 +14,9 @@ build-core: GIT_STRATEGY: clone DOCKER_FILE_PATH: "core/docker/Dockerfile" DATAFED_HARBOR_REGISTRY: "$REGISTRY" # needed by c_harbor_artifact_count - BUILD_INTERMEDIATE: "FALSE" + BUILD_INTERMEDIATE: "TRUE" + INTERMEDIATE_TARGET: "core-build" # Name of the layer in the dockerfile + INTERMEDIATE_LAYER_NAME: "build" tags: - ci-datafed-core - docker diff --git a/.gitlab/build/retag_core_image.yml b/.gitlab/build/retag_core_image.yml index 8b719df82..6042b367b 100644 --- a/.gitlab/build/retag_core_image.yml +++ b/.gitlab/build/retag_core_image.yml @@ -13,6 +13,8 @@ retag-image: COMPONENT: "core" GIT_STRATEGY: clone DATAFED_HARBOR_REGISTRY: "$REGISTRY" # needed by c_harbor_artifact_count - BUILD_INTERMEDIATE: "FALSE" + BUILD_INTERMEDIATE: "TRUE" + INTERMEDIATE_TARGET: "core-build" # Name of the layer in the dockerfile + INTERMEDIATE_LAYER_NAME: "build" tags: - docker diff --git a/.gitlab/stage_unit.yml b/.gitlab/stage_unit.yml index 8a246e63e..bd14d9e33 100644 --- a/.gitlab/stage_unit.yml +++ b/.gitlab/stage_unit.yml @@ -128,7 +128,6 @@ run-authz-unit-job: - chmod +x run_globus.sh - ./run_globus.sh - run-core-unit-job: needs: ["run-core-build-job"] stage: unit From 24b056954e5dec3e2f224b16159336fb0530ef87 Mon Sep 17 00:00:00 2001 From: JoshuaSBrown Date: Wed, 25 Mar 2026 06:36:31 -0400 Subject: [PATCH 5/9] feat: enable building of core integration tests --- core/docker/Dockerfile | 2 +- tests/mock/CMakeLists.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/core/docker/Dockerfile b/core/docker/Dockerfile index eb1ebc36e..dde9e0e7d 100644 --- a/core/docker/Dockerfile +++ b/core/docker/Dockerfile @@ -59,7 +59,7 @@ RUN ${DATAFED_DEPENDENCIES_ROOT}/scripts/generate_dependencies_config.sh && \ -DBUILD_PYTHON_CLIENT=False \ -DBUILD_FOXX=False \ -DENABLE_UNIT_TESTS=True \ - -DENABLE_INTEGRATION_TESTS=False + -DENABLE_INTEGRATION_TESTS=True RUN ${DATAFED_DEPENDENCIES_INSTALL_PATH}/bin/cmake --build build -j 8 RUN ${DATAFED_DEPENDENCIES_INSTALL_PATH}/bin/cmake --build build --target install diff --git a/tests/mock/CMakeLists.txt b/tests/mock/CMakeLists.txt index 0e8b7eeb0..759f52d49 100644 --- a/tests/mock/CMakeLists.txt +++ b/tests/mock/CMakeLists.txt @@ -8,7 +8,7 @@ foreach(PROG file(GLOB ${PROG}_SOURCES ${PROG}*.cpp) add_executable(mock_liveness_${PROG} ${${PROG}_SOURCES}) - target_link_libraries(mock_liveness_${PROG} PUBLIC ${DATAFED_BOOST_LIBRARIES} ${DATAFED_GSSAPI_LIBRARIES} ${DATAFED_GLOBUS_COMMON_LIBRARIES} common ) + target_link_libraries(mock_liveness_${PROG} PUBLIC ${DATAFED_BOOST_LIBRARIES} common ) if(BUILD_SHARED_LIBS) target_compile_definitions(mock_liveness_${PROG} PRIVATE BOOST_TEST_DYN_LINK) endif() From 51ced0ca67b6584287e0364f0ffc5c7816c8d3ec Mon Sep 17 00:00:00 2001 From: JoshuaSBrown Date: Wed, 25 Mar 2026 07:01:44 -0400 Subject: [PATCH 6/9] fix: add intermediate to core image check. --- .gitlab/stage_image_check.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab/stage_image_check.yml b/.gitlab/stage_image_check.yml index d29b8209f..5cf061748 100644 --- a/.gitlab/stage_image_check.yml +++ b/.gitlab/stage_image_check.yml @@ -18,7 +18,8 @@ check-core-image: variables: PROJECT: "datafed" COMPONENT: "core" - BUILD_INTERMEDIATE: "FALSE" + BUILD_INTERMEDIATE: "TRUE" + INTERMEDIATE_LAYER_NAME: "build" WATCHED_PATHS: "docker scripts core common CMakeLists.txt cmake .gitlab-ci.yml" check-repo-image: From b20add2ad6ff1823dde2bdeedb38ef9b5450e41c Mon Sep 17 00:00:00 2001 From: JoshuaSBrown Date: Wed, 25 Mar 2026 07:23:08 -0400 Subject: [PATCH 7/9] fix: add missing test labels --- tests/CMakeLists.txt | 4 ++-- tests/mock/CMakeLists.txt | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 1677976af..cf8f01d48 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -27,8 +27,8 @@ if( ENABLE_INTEGRATION_TESTS ) WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/tests/mock_core ) - set_tests_properties(start_mock PROPERTIES FIXTURES_SETUP FIX_MOCK) - set_tests_properties(stop_mock PROPERTIES FIXTURES_CLEANUP FIX_MOCK) + set_tests_properties(start_mock PROPERTIES FIXTURES_SETUP FIX_MOCK LABELS "integration") + set_tests_properties(stop_mock PROPERTIES FIXTURES_CLEANUP FIX_MOCK LABELS "integration")) add_subdirectory(mock_core) add_subdirectory(mock) diff --git a/tests/mock/CMakeLists.txt b/tests/mock/CMakeLists.txt index 0e8b7eeb0..038f16943 100644 --- a/tests/mock/CMakeLists.txt +++ b/tests/mock/CMakeLists.txt @@ -19,6 +19,6 @@ foreach(PROG COMMAND ${CMAKE_CURRENT_BINARY_DIR}/mock_liveness_${PROG} WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/tests/mock_core ) - set_tests_properties(mock_liveness_${PROG} PROPERTIES FIXTURES_REQUIRED FIX_MOCK) + set_tests_properties(mock_liveness_${PROG} PROPERTIES FIXTURES_REQUIRED FIX_MOCK LABELS "integration") endforeach(PROG) From 2cc37f93062cbdad46f79f48cebcf8e9a76137d7 Mon Sep 17 00:00:00 2001 From: JoshuaSBrown Date: Wed, 25 Mar 2026 07:24:28 -0400 Subject: [PATCH 8/9] refactor: add core build intermediate stage to compose build --- scripts/compose_build_images.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/scripts/compose_build_images.sh b/scripts/compose_build_images.sh index 1536367b7..ee771de5f 100755 --- a/scripts/compose_build_images.sh +++ b/scripts/compose_build_images.sh @@ -73,6 +73,13 @@ if [[ "$BUILD_METADATA" == "TRUE" ]]; then --build-arg BASE_IMAGE=$BASE_IMAGE \ -t datafed-runtime:latest fi + docker build -f \ + "${PROJECT_ROOT}/core/docker/Dockerfile" \ + --build-arg DEPENDENCIES="datafed-dependencies:latest" \ + --build-arg RUNTIME="datafed-runtime:latest" \ + --target core-build \ + "${PROJECT_ROOT}" \ + -t datafed-core-build:latest docker build -f \ "${PROJECT_ROOT}/core/docker/Dockerfile" \ --build-arg DEPENDENCIES="datafed-dependencies:latest" \ From 7cf7dcae5951ec55c75862db4e71cdf4211105f1 Mon Sep 17 00:00:00 2001 From: JoshuaSBrown Date: Wed, 25 Mar 2026 14:35:51 -0400 Subject: [PATCH 9/9] refactor: Update CMakeLists.txt to include client handler for schemas. --- core/server/tests/integration/CMakeLists.txt | 80 +++++++++++++++++++- 1 file changed, 76 insertions(+), 4 deletions(-) diff --git a/core/server/tests/integration/CMakeLists.txt b/core/server/tests/integration/CMakeLists.txt index e26556b19..f46bc9315 100644 --- a/core/server/tests/integration/CMakeLists.txt +++ b/core/server/tests/integration/CMakeLists.txt @@ -11,11 +11,20 @@ set(MOCK_SCHEMA_PORT "4010" set(MOCK_SCHEMA_PULL "true" CACHE STRING "Pull the mock image before starting (set to 'false' for offline/local builds)") +set(ARANGO_IMAGE "savannah.ornl.gov/datafed/arango-foxx:latest" + CACHE STRING "Docker image for the ArangoDB test instance (with Foxx services)") +set(ARANGO_PORT "8529" + CACHE STRING "Host port for the ArangoDB test container") +set(ARANGO_ROOT_PASS "test" + CACHE STRING "Root password for the ArangoDB test instance") +set(ARANGO_PULL "true" + CACHE STRING "Pull the ArangoDB image before starting (set to 'false' for offline/local builds)") + # ── Check for Docker (needed by SchemaAPIClient tests) ─────────────────────── find_program(DOCKER_EXECUTABLE docker) if(NOT DOCKER_EXECUTABLE) - message(STATUS "Docker not found — SchemaAPIClient integration tests will be skipped") + message(STATUS "Docker not found — integration tests requiring containers will be skipped") endif() # ── Compile and register all integration tests ────────────────────────────── @@ -23,6 +32,7 @@ endif() foreach(PROG test_SchemaServiceFactory test_SchemaAPIClient + test_SchemaHandler ) file(GLOB ${PROG}_SOURCES ${PROG}*.cpp) add_executable(integration_${PROG} ${${PROG}_SOURCES}) @@ -52,6 +62,10 @@ endforeach(PROG) if(DOCKER_EXECUTABLE) + # ======================================================================== + # Mock Schema API (Prism) fixture — used by SchemaAPIClient + # ======================================================================== + set(_mock_schema_env "MOCK_SCHEMA_IMAGE=${MOCK_SCHEMA_IMAGE}" "MOCK_SCHEMA_PORT=${MOCK_SCHEMA_PORT}" @@ -59,7 +73,7 @@ if(DOCKER_EXECUTABLE) "CONTAINER_NAME=datafed-mock-schema" ) - set(_test_env + set(_mock_schema_test_env "SCHEMA_API_TEST_URL=http://localhost:${MOCK_SCHEMA_PORT}" ) @@ -91,14 +105,72 @@ if(DOCKER_EXECUTABLE) set_tests_properties(integration_test_SchemaAPIClient PROPERTIES FIXTURES_REQUIRED MockSchemaAPI - ENVIRONMENT "${_test_env}" + ENVIRONMENT "${_mock_schema_test_env}" LABELS "integration" ) if(ENABLE_MEMORY_TESTS) set_tests_properties(memory_test_SchemaAPIClient PROPERTIES FIXTURES_REQUIRED MockSchemaAPI - ENVIRONMENT "${_test_env}" + ENVIRONMENT "${_mock_schema_test_env}" + LABELS "integration;memory" + ) + endif() + + # ======================================================================== + # ArangoDB fixture — used by SchemaHandler + # ======================================================================== + + set(_arango_env + "ARANGO_IMAGE=${ARANGO_IMAGE}" + "ARANGO_PORT=${ARANGO_PORT}" + "ARANGO_ROOT_PASS=${ARANGO_ROOT_PASS}" + "ARANGO_PULL=${ARANGO_PULL}" + "ARANGO_CONTAINER=datafed-test-arango" + ) + + set(_arango_test_env + "DATAFED_TEST_ARANGO_URL=http://localhost:${ARANGO_PORT}" + "DATAFED_TEST_ARANGO_USER=root" + "DATAFED_TEST_ARANGO_PASS=${ARANGO_ROOT_PASS}" + ) + + # ── Fixture setup: start ArangoDB container ────────────────────────── + + add_test( + NAME arango_start + COMMAND bash ${CMAKE_CURRENT_SOURCE_DIR}/start_test_arango.sh + ) + set_tests_properties(arango_start PROPERTIES + FIXTURES_SETUP TestArangoDB + ENVIRONMENT "${_arango_env}" + LABELS "fixture;integration" + ) + + # ── Fixture cleanup: stop and remove container ─────────────────────── + + add_test( + NAME arango_stop + COMMAND bash ${CMAKE_CURRENT_SOURCE_DIR}/stop_test_arango.sh + ) + set_tests_properties(arango_stop PROPERTIES + FIXTURES_CLEANUP TestArangoDB + ENVIRONMENT "${_arango_env}" + LABELS "fixture;integration" + ) + + # ── Attach fixture to SchemaHandler tests ──────────────────────────── + + set_tests_properties(integration_test_SchemaHandler PROPERTIES + FIXTURES_REQUIRED TestArangoDB + ENVIRONMENT "${_arango_test_env}" + LABELS "integration" + ) + + if(ENABLE_MEMORY_TESTS) + set_tests_properties(memory_test_SchemaHandler PROPERTIES + FIXTURES_REQUIRED TestArangoDB + ENVIRONMENT "${_arango_test_env}" LABELS "integration;memory" ) endif()