From f35c0e4901f7c09eab166d6b035ffe99b6cfa19f Mon Sep 17 00:00:00 2001 From: Garrett Birkel Date: Wed, 3 Dec 2025 20:19:58 -0800 Subject: [PATCH 01/15] Minor rename for clarity. --- orchestration/_tests/test_config.py | 6 +++--- orchestration/config.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/orchestration/_tests/test_config.py b/orchestration/_tests/test_config.py index 608b5400..bca60c6e 100644 --- a/orchestration/_tests/test_config.py +++ b/orchestration/_tests/test_config.py @@ -8,9 +8,9 @@ def test_config(): config_file = Path(__file__).parent / "test_config.yml" with MonkeyPatch.context(): - globus_config = read_config(config_file=config_file) - assert globus_config + config = read_config(config_file=config_file) + assert config assert ( - globus_config["globus"]["globus_endpoints"]["test_endpoint"]["root_path"] + config["globus"]["globus_endpoints"]["test_endpoint"]["root_path"] == "/data" ) diff --git a/orchestration/config.py b/orchestration/config.py index 79083375..6abf5fc3 100644 --- a/orchestration/config.py +++ b/orchestration/config.py @@ -12,8 +12,8 @@ def get_config(): def read_config(config_file="config.yml"): with open(config_file, "r") as end_file: - globus_config = yaml.safe_load(end_file) - return expand_environment_variables(globus_config) + config = yaml.safe_load(end_file) + return expand_environment_variables(config) def expand_environment_variables(config): From ab70fde4ea5a6823b56707c6af680127c2213166 Mon Sep 17 00:00:00 2001 From: Garrett Birkel Date: Wed, 3 Dec 2025 20:20:34 -0800 Subject: [PATCH 02/15] Reordering requirements --- requirements.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/requirements.txt b/requirements.txt index 24b8e9e4..40819c89 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,7 @@ +authlib +globus-compute-sdk @ git+https://github.com/globus/globus-compute.git@d1731340074be56861ec91d732bdff44f8e2b46e#subdirectory=compute_sdk globus-sdk>=3.0 +griffe>=0.49.0,<2.0.0 h5py httpx>=0.22.0 mkdocs @@ -6,13 +9,10 @@ mkdocs-material mkdocs-mermaid2-plugin numpy>=1.26.4 pillow +prometheus_client==0.21.1 pydantic==2.11 python-dotenv prefect==2.20.17 pyscicat pyyaml -authlib sfapi_client -globus-compute-sdk @ git+https://github.com/globus/globus-compute.git@d1731340074be56861ec91d732bdff44f8e2b46e#subdirectory=compute_sdk -griffe>=0.49.0,<2.0.0 -prometheus_client==0.21.1 From 8d644be43f4a4fc0dd1db38b202fc3fad7666971 Mon Sep 17 00:00:00 2001 From: Garrett Birkel Date: Wed, 3 Dec 2025 20:22:33 -0800 Subject: [PATCH 03/15] Adding in scicat_beamline requirement and creating a pyproject.toml file. --- pyproject.toml | 39 +++++++++++++++++++++++++++++++++++++++ requirements.txt | 1 + 2 files changed, 40 insertions(+) create mode 100644 pyproject.toml diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..1a9b8262 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,39 @@ +[project] +name = "splash-flows" +version = "0.1.0" +description = "ALS configuration and code for Prefect workflows to move data and run computing tasks" +readme = "README.md" +requires-python = ">=3.12" +dependencies = [ + "authlib", + "globus-compute-sdk @ git+https://github.com/globus/globus-compute.git@d1731340074be56861ec91d732bdff44f8e2b46e#subdirectory=compute_sdk", + "globus-sdk>=3.0", + "griffe>=0.49.0,<2.0.0", + "h5py", + "httpx>=0.22.0", + "mkdocs", + "mkdocs-material", + "mkdocs-mermaid2-plugin", + "numpy>=1.26.4", + "pillow", + "prometheus_client==0.21.1", + "pydantic==2.11", + "python-dotenv", + "prefect==2.20.17", + "pyscicat @ git+https://github.com/SciCatProject/pyscicat.git@main#subdirectory=pyscicat", + "pyyaml", + "sfapi_client", + "scicat_beamline @ git+https://github.com/als-computing/scicat_beamline.git@main#subdirectory=scicat_beamline" +] + +[build-system] +requires = [ + "black", + "flake8", + "freezegun", + "pytest", + "pytest-mock" +] +build-backend = "setuptools.build_meta" + + diff --git a/requirements.txt b/requirements.txt index 40819c89..4324f553 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,3 +16,4 @@ prefect==2.20.17 pyscicat pyyaml sfapi_client +scicat_beamline @ git+https://github.com/als-computing/scicat_beamline.git@main#subdirectory=scicat_beamline From d6d1ccf7e8755d9484f3c61fc9feb6179c9828cc Mon Sep 17 00:00:00 2001 From: Garrett Birkel Date: Thu, 4 Dec 2025 14:37:27 -0800 Subject: [PATCH 04/15] 832 ingest code moved over to scicat_beamline, and named "als_832_dx_4". Updating "ingest_tomo832" to use call the SciCat ingester module. --- orchestration/flows/bl832/ingest_tomo832.py | 424 +------------------- 1 file changed, 3 insertions(+), 421 deletions(-) diff --git a/orchestration/flows/bl832/ingest_tomo832.py b/orchestration/flows/bl832/ingest_tomo832.py index 096039e2..d6c6c459 100644 --- a/orchestration/flows/bl832/ingest_tomo832.py +++ b/orchestration/flows/bl832/ingest_tomo832.py @@ -1,415 +1,10 @@ -from datetime import datetime -import json -import logging -import os -from pathlib import Path -import re -from typing import Any, Dict, List - -import h5py -from pyscicat.client import ScicatClient -from pyscicat.model import ( - Attachment, - CreateDatasetOrigDatablockDto, - Datablock, - DataFile, - RawDataset, - DatasetType, - Ownable, -) - -from orchestration.flows.scicat.utils import ( - build_search_terms, - build_thumbnail, - calculate_access_controls, - encode_image_2_thumbnail, - Issue, - NPArrayEncoder, - Severity -) - -DEFAULT_USER = "8.3.2" # In case there's not proposal number -UNKNOWN_EMAIL = "unknown@example.com" -ingest_spec = "als832_dx_4" # "als832_dx_3" - -logger = logging.getLogger("scicat_ingest") - - -def ingest( - scicat_client: ScicatClient, - file_path: str, - issues: List[Issue], - log_level: str = "INFO", -) -> str: - """Ingests a file into scicat - - Ingestion to takes a "best effort" stance to ingestion. Along the way, - many things can go wrong. Rather than failing the entire ingestion, we - collect issues and return them to the caller. These issues are updated - in the input issues list. - - Parameters - ---------- - scicat_client : ScicatClient - client to talk to the scicat server - file_path : str - Path to find the file to ingest - issues : List[Issue] - Issues where problems are recorded - - Returns - ------- - str - Dataset id of the new - """ - logger.setLevel(log_level) - INGEST_STORAGE_ROOT_PATH = os.getenv("INGEST_STORAGE_ROOT_PATH") - INGEST_SOURCE_ROOT_PATH = os.getenv("INGEST_SOURCE_ROOT_PATH") - - if not INGEST_SOURCE_ROOT_PATH or not INGEST_SOURCE_ROOT_PATH: - raise ValueError( - "INGEST_STORAGE_ROOT_PATH and INGEST_SOURCE_ROOT_PATH must be set" - ) - - with h5py.File(file_path, "r") as file: - file_path = Path(file_path) - scicat_metadata = _extract_fields(file, scicat_metadata_keys, issues) - scientific_metadata = _extract_fields(file, scientific_metadata_keys, issues) - scientific_metadata["data_sample"] = _get_data_sample(file) - encoded_scientific_metadata = json.loads( - json.dumps(scientific_metadata, cls=NPArrayEncoder) - ) - access_controls = calculate_access_controls( - DEFAULT_USER, - scicat_metadata.get("/measurement/sample/experiment/beamline"), - scicat_metadata.get("/measurement/sample/experiment/proposal"), - ) - logger.info( - f"Access controls for {file_path} access_groups: {access_controls.get('accessroups')} " - f"owner_group: {access_controls.get('owner_group')}" - ) - - ownable = Ownable( - ownerGroup=access_controls["owner_group"], - accessGroups=access_controls["access_groups"], - ) - dataset_id = upload_raw_dataset( - scicat_client, - file_path, - scicat_metadata, - encoded_scientific_metadata, - ownable, - ) - upload_data_block( - scicat_client, - file_path, - dataset_id, - INGEST_STORAGE_ROOT_PATH, - INGEST_SOURCE_ROOT_PATH) - - thumbnail_file = build_thumbnail(file["/exchange/data"][0]) - encoded_thumbnail = encode_image_2_thumbnail(thumbnail_file) - upload_attachment( - scicat_client, - encoded_thumbnail, - dataset_id, - ownable) - - return dataset_id - - -def upload_raw_dataset( - scicat_client: ScicatClient, - file_path: Path, - scicat_metadata: Dict, - scientific_metadata: Dict, - ownable: Ownable, -) -> str: - "Creates a dataset object" - file_size = get_file_size(file_path) - file_mod_time = get_file_mod_time(file_path) - file_name = scicat_metadata.get("/measurement/sample/file_name") - description = build_search_terms(file_name) - appended_keywords = description.split() - logger.info(f"email: {scicat_metadata.get('/measurement/sample/experimenter/email')}") - dataset = RawDataset( - owner=scicat_metadata.get("/measurement/sample/experiment/pi") or "Unknown", - contactEmail=clean_email(scicat_metadata.get("/measurement/sample/experimenter/email")) - or "unknown@example.com", - creationLocation=scicat_metadata.get("/measurement/instrument/instrument_name") - or "Unknown", - datasetName=file_name, - type=DatasetType.raw, - instrumentId=scicat_metadata.get("/measurement/instrument/instrument_name") - or "Unknown", - proposalId=scicat_metadata.get("/measurement/sample/experiment/proposal"), - dataFormat="DX", - principalInvestigator=scicat_metadata.get("/measurement/sample/experiment/pi") - or "Unknown", - sourceFolder=str(file_path.parent), - size=file_size, - scientificMetadata=scientific_metadata, - sampleId=description, - isPublished=False, - description=description, - keywords=appended_keywords, - creationTime=file_mod_time, - **ownable.dict(), - ) - logger.debug(f"dataset: {dataset}") - dataset_id = scicat_client.upload_new_dataset(dataset) - return dataset_id - - -def create_data_files(file_path: Path, storage_path: str) -> List[DataFile]: - "Collects all fits files" - datafiles = [] - datafile = DataFile( - path=storage_path, - size=get_file_size(file_path), - time=get_file_mod_time(file_path), - type="RawDatasets", - ) - datafiles.append(datafile) - return datafiles - - -def upload_data_block( - scicat_client: ScicatClient, - file_path: Path, - dataset_id: str, - storage_root_path: str, - source_root_path: str -) -> Datablock: - "Creates a datablock of files" - # calcularte the path where the file will as known to SciCat - storage_path = str(file_path).replace(source_root_path, storage_root_path) - datafiles = create_data_files(file_path, storage_path) - - datablock = CreateDatasetOrigDatablockDto( - size=get_file_size(file_path), - dataFileList=datafiles - ) - return scicat_client.upload_dataset_origdatablock(dataset_id, datablock) - - -def upload_attachment( - scicat_client: ScicatClient, - encoded_thumnbnail: str, - dataset_id: str, - ownable: Ownable, -) -> Attachment: - "Creates a thumbnail png" - attachment = Attachment( - datasetId=dataset_id, - thumbnail=encoded_thumnbnail, - caption="raw image", - **ownable.dict(), - ) - scicat_client.upload_attachment(attachment) - - -def get_file_size(file_path: Path) -> int: - return file_path.lstat().st_size - - -def get_file_mod_time(file_path: Path) -> str: - return datetime.fromtimestamp(file_path.lstat().st_mtime).isoformat() - - -def _extract_fields(file, keys, issues) -> Dict[str, Any]: - metadata = {} - for md_key in keys: - dataset = file.get(md_key) - if not dataset: - issues.append( - Issue(msg=f"dataset not found {md_key}", severity=Severity.warning) - ) - continue - metadata[md_key] = _get_dataset_value(file[md_key]) - return metadata - - -def _get_dataset_value(data_set): - logger.debug(f"{data_set} {data_set.dtype}") - try: - if "S" in data_set.dtype.str: - if data_set.shape == (1,): - return data_set.asstr()[0] - elif data_set.shape == (): - return data_set[()].decode("utf-8") - else: - return list(data_set.asstr()) - else: - if data_set.maxshape == (1,): - logger.debug(f"{data_set} {data_set[()][0]}") - return data_set[()][0] - else: - logger.debug(f"{data_set} {data_set[()]}") - return data_set[()] - except Exception: - logger.exception("Exception extracting dataset value") - return None - - -def _get_data_sample(file, sample_size=10): - data_sample = {} - for key in data_sample_keys: - data_array = file.get(key) - if not data_array: - continue - step_size = int(len(data_array) / sample_size) - if step_size == 0: - step_size = 1 - sample = data_array[0::step_size] - data_sample[key] = sample - - return data_sample - - -# TODO: Move clean_email() to the scicat_beamline repository -# https://github.com/als-computing/scicat_beamline -def clean_email(email: any) -> str: - """ - Clean the provided email address. - - This function ensures that the input is a valid email address. - It returns a default email if: - - The input is not a string, - - The input is empty after stripping, - - The input equals "NONE" (case-insensitive), or - - The input does not contain an "@" symbol. - - Parameters - ---------- - email : any - The raw email value extracted from metadata. - - Returns - ------- - str - A cleaned email address if valid, otherwise the default unknown email. - - Example - ------- - >>> clean_email(" user@example.com ") - 'user@example.com' - >>> clean_email("garbage") - 'unknown@example.com' - >>> clean_email(None) - 'unknown@example.com' - """ - # Check that the email is a string - if not isinstance(email, str): - logger.info(f"Input email is not a string. Returning {UNKNOWN_EMAIL}") - return UNKNOWN_EMAIL - - # Remove surrounding whitespace - cleaned = email.strip() - - # Remove leading/trailing quotes, commas, and whitespace - cleaned = re.sub(r'^["\'\s,]+|["\'\s,]+$', '', email) - - # Fallback if the email is empty, equals "NONE", or lacks an "@" symbol - if not cleaned or cleaned.upper() == "NONE" or "@" not in cleaned: - logger.info(f"Invalid email address. Returning {UNKNOWN_EMAIL}") - return UNKNOWN_EMAIL - - # Optionally, remove spaces from inside the email (typically invalid in an email address) - cleaned = cleaned.replace(" ", "") - - # Final verification: ensure that the cleaned email contains "@". - if "@" not in cleaned: - logger.info(f"Invalid email address: {cleaned}. Returning {UNKNOWN_EMAIL}") - return UNKNOWN_EMAIL - - return cleaned - - -scicat_metadata_keys = [ - "/measurement/instrument/instrument_name", - "/measurement/sample/experiment/beamline", - "/measurement/sample/experiment/experiment_lead", - "/measurement/sample/experiment/pi", - "/measurement/sample/experiment/proposal", - "/measurement/sample/experimenter/email", - "/measurement/sample/experimenter/name", - "/measurement/sample/file_name", -] - -scientific_metadata_keys = [ - "/measurement/instrument/attenuator/setup/filter_y", - "/measurement/instrument/camera_motor_stack/setup/tilt_motor", - "/measurement/instrument/detection_system/objective/camera_objective", - "/measurement/instrument/detection_system/scintillator/scintillator_type", - "/measurement/instrument/detector/binning_x", - "/measurement/instrument/detector/binning_y", - "/measurement/instrument/detector/dark_field_value", - "/measurement/instrument/detector/delay_time", - "/measurement/instrument/detector/dimension_x", - "/measurement/instrument/detector/dimension_y", - "/measurement/instrument/detector/model", - "/measurement/instrument/detector/pixel_size", - "/measurement/instrument/detector/temperature", - "/measurement/instrument/monochromator/setup/Z2", - # NOTE: These are commented out because they are no longer present in the h5 file as of March 25, 2025 - # Keeping them commented out in case they are needed in the future - # "/measurement/instrument/monochromator/setup/temperature_tc2", - # "/measurement/instrument/monochromator/setup/temperature_tc3", - # "/measurement/instrument/slits/setup/hslits_A_Door", - # "/measurement/instrument/slits/setup/hslits_A_Wall", - "/measurement/instrument/slits/setup/hslits_center", - "/measurement/instrument/slits/setup/hslits_size", - "/measurement/instrument/slits/setup/vslits_Lead_Flag", - "/measurement/instrument/source/source_name", - "/process/acquisition/dark_fields/dark_num_avg_of", - "/process/acquisition/dark_fields/num_dark_fields", - "/process/acquisition/flat_fields/i0_move_x", - "/process/acquisition/flat_fields/i0_move_y", - "/process/acquisition/flat_fields/i0cycle", - "/process/acquisition/flat_fields/num_flat_fields", - "/process/acquisition/flat_fields/usebrightexpose", - "/process/acquisition/mosaic/tile_xmovedist", - "/process/acquisition/mosaic/tile_xnumimg", - "/process/acquisition/mosaic/tile_xorig", - "/process/acquisition/mosaic/tile_xoverlap", - "/process/acquisition/mosaic/tile_ymovedist", - "/process/acquisition/mosaic/tile_ynumimg", - "/process/acquisition/mosaic/tile_yorig", - "/process/acquisition/mosaic/tile_yoverlap", - "/process/acquisition/name", - "/process/acquisition/rotation/blur_limit", - "/process/acquisition/rotation/blur_limit", - "/process/acquisition/rotation/multiRev", - "/process/acquisition/rotation/nhalfCir", - "/process/acquisition/rotation/num_angles", - "/process/acquisition/rotation/range", -] - -data_sample_keys = [ - "/measurement/instrument/sample_motor_stack/setup/axis1pos", - "/measurement/instrument/sample_motor_stack/setup/axis2pos", - "/measurement/instrument/sample_motor_stack/setup/sample_x", - "/measurement/instrument/sample_motor_stack/setup/axis5pos", - "/measurement/instrument/camera_motor_stack/setup/camera_elevation", - "/measurement/instrument/source/current", - "/measurement/instrument/camera_motor_stack/setup/camera_distance", - "/measurement/instrument/source/beam_intensity_incident", - "/measurement/instrument/monochromator/energy", - "/measurement/instrument/detector/exposure_time", - "/measurement/instrument/time_stamp", - "/measurement/instrument/monochromator/setup/turret2", - "/measurement/instrument/monochromator/setup/turret1", -] - - def test_ingest_raw_tomo() -> bool: from orchestration.flows.scicat.ingest import ingest_dataset - TOMO_INGESTOR_MODULE = "orchestration.flows.bl832.ingest_tomo832" + INGESTER_SPEC = "als_832_dx_4" file_path = "examples/tomo_scan_no_email.h5" - print(f"Ingesting {file_path} with {TOMO_INGESTOR_MODULE}") + print(f"Ingesting {file_path} with {INGESTER_SPEC}") try: - ingest_dataset(file_path, TOMO_INGESTOR_MODULE) + ingest_dataset(file_path, INGESTER_SPEC) return True except Exception as e: print(f"SciCat ingest failed with {e}") @@ -417,18 +12,5 @@ def test_ingest_raw_tomo() -> bool: if __name__ == "__main__": - # ingest( - # ScicatClient( - # # "http://localhost:3000/api/v3", - # os.environ.get("SCICAT_API_URL"), - # None, - # os.environ.get("SCICAT_INGEST_USER"), - # os.environ.get("SCICAT_INGEST_PASSWORD"), - # ), - # "/Users/dylanmcreynolds/data/beamlines/8.3.2/raw/" - # "20231013_065251_MSB_Book1_Proj77_Cell3_Gen2_Li_R2G_FastCharge_DuringCharge0.h5", - # [], - # log_level="DEBUG", - # ) test_ingest_raw_tomo() From 566795fbc85ea245a75094eba5cbc4cfbc56258c Mon Sep 17 00:00:00 2001 From: Garrett Birkel Date: Wed, 17 Dec 2025 14:31:05 -0800 Subject: [PATCH 05/15] Don't need pyscicat directly at this point. Updated reference to scicat-beamline-ingestion. --- pyproject.toml | 6 +++--- requirements.txt | 3 +-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 1a9b8262..f83c94ce 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,10 +20,9 @@ dependencies = [ "pydantic==2.11", "python-dotenv", "prefect==2.20.17", - "pyscicat @ git+https://github.com/SciCatProject/pyscicat.git@main#subdirectory=pyscicat", "pyyaml", "sfapi_client", - "scicat_beamline @ git+https://github.com/als-computing/scicat_beamline.git@main#subdirectory=scicat_beamline" + "scicat_beamline_ingestion @ git+https://github.com/als-computing/scicat_beamline_ingestion.git@2025/12/10-flow" ] [build-system] @@ -32,7 +31,8 @@ requires = [ "flake8", "freezegun", "pytest", - "pytest-mock" + "pytest-mock", + "setuptools" ] build-backend = "setuptools.build_meta" diff --git a/requirements.txt b/requirements.txt index 4324f553..a00c2fc9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,7 +13,6 @@ prometheus_client==0.21.1 pydantic==2.11 python-dotenv prefect==2.20.17 -pyscicat pyyaml sfapi_client -scicat_beamline @ git+https://github.com/als-computing/scicat_beamline.git@main#subdirectory=scicat_beamline +scicat-beamline-ingestion @ git+https://github.com/als-computing/scicat_beamline_ingestion.git@2025/12/10-flow From ad052ee6ba7738309a46ca1a798010cd6ec38b9b Mon Sep 17 00:00:00 2001 From: Garrett Birkel Date: Wed, 17 Dec 2025 16:34:08 -0800 Subject: [PATCH 06/15] A bit more requirements shuffling --- pyproject.toml | 8 ++++---- requirements.txt | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index f83c94ce..725f73cb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ name = "splash-flows" version = "0.1.0" description = "ALS configuration and code for Prefect workflows to move data and run computing tasks" readme = "README.md" -requires-python = ">=3.12" +requires-python = ">=3.11" dependencies = [ "authlib", "globus-compute-sdk @ git+https://github.com/globus/globus-compute.git@d1731340074be56861ec91d732bdff44f8e2b46e#subdirectory=compute_sdk", @@ -16,13 +16,13 @@ dependencies = [ "mkdocs-mermaid2-plugin", "numpy>=1.26.4", "pillow", + "prefect==3.4.2", "prometheus_client==0.21.1", "pydantic==2.11", "python-dotenv", - "prefect==2.20.17", "pyyaml", - "sfapi_client", - "scicat_beamline_ingestion @ git+https://github.com/als-computing/scicat_beamline_ingestion.git@2025/12/10-flow" + "scicat_beamline_ingestion @ git+https://github.com/als-computing/scicat_beamline_ingestion.git@main", + "sfapi_client" ] [build-system] diff --git a/requirements.txt b/requirements.txt index bf668ddc..17449efb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,10 +9,10 @@ mkdocs-material mkdocs-mermaid2-plugin numpy>=1.26.4 pillow +prefect==3.4.2 prometheus_client==0.21.1 pydantic==2.11 python-dotenv -prefect==3.4.2 pyyaml +scicat-beamline-ingestion @ git+https://github.com/als-computing/scicat_beamline_ingestion.git@main sfapi_client -scicat-beamline-ingestion @ git+https://github.com/als-computing/scicat_beamline_ingestion.git@2025/12/10-flow From f9c80794bcb7a4e3a6c079b1a6c667833b9fe026 Mon Sep 17 00:00:00 2001 From: Garrett Birkel Date: Wed, 17 Dec 2025 16:59:43 -0800 Subject: [PATCH 07/15] Not using this code path now --- orchestration/flows/bl832/ingest_tomo832.py | 16 ---------------- 1 file changed, 16 deletions(-) delete mode 100644 orchestration/flows/bl832/ingest_tomo832.py diff --git a/orchestration/flows/bl832/ingest_tomo832.py b/orchestration/flows/bl832/ingest_tomo832.py deleted file mode 100644 index d6c6c459..00000000 --- a/orchestration/flows/bl832/ingest_tomo832.py +++ /dev/null @@ -1,16 +0,0 @@ -def test_ingest_raw_tomo() -> bool: - from orchestration.flows.scicat.ingest import ingest_dataset - INGESTER_SPEC = "als_832_dx_4" - file_path = "examples/tomo_scan_no_email.h5" - print(f"Ingesting {file_path} with {INGESTER_SPEC}") - try: - ingest_dataset(file_path, INGESTER_SPEC) - return True - except Exception as e: - print(f"SciCat ingest failed with {e}") - return False - - -if __name__ == "__main__": - - test_ingest_raw_tomo() From da04954f7bd41b0ea7c7d4ee4dd1d5f8b0f032e3 Mon Sep 17 00:00:00 2001 From: Garrett Birkel Date: Wed, 17 Dec 2025 17:00:06 -0800 Subject: [PATCH 08/15] Testing the function underlying the import dlow (has a different call signature) --- orchestration/_tests/test_scicat.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/orchestration/_tests/test_scicat.py b/orchestration/_tests/test_scicat.py index 6cc68cdb..622066d7 100644 --- a/orchestration/_tests/test_scicat.py +++ b/orchestration/_tests/test_scicat.py @@ -4,14 +4,16 @@ import sys import types from typing import List +from pathlib import Path from pytest_mock import MockFixture +from scicat_beamline_ingestion.flows import scicat_ingest_flow from orchestration.flows.scicat.utils import NPArrayEncoder, build_search_terms, calculate_access_controls, Issue -from orchestration.flows.scicat.ingest import ingest_dataset_task from orchestration.flows.bl832.ingest_tomo832 import clean_email, UNKNOWN_EMAIL +# TODO: Move these to scicat_beamline_ingestion def test_clean_email_valid(): # Remove surrounding whitespace. assert clean_email(" user@example.com ") == "user@example.com" @@ -125,7 +127,13 @@ def warning(self, *args, **kwargs): pass def error(self, *args, **kwargs): pass -def dummy_ingest(scicat_client, file_path, issues: List[Issue]): +def dummy_ingest( + scicat_client, + owner_username: str, + file_path: Path, + thumbnail_dir: Path, + issues: List[Issue], +): issues.clear() return "dummy_dataset_id" @@ -133,7 +141,7 @@ def dummy_ingest(scicat_client, file_path, issues: List[Issue]): def test_ingest_dataset_task(mocker: MockFixture): # Set environment variables. mocker.patch.dict(os.environ, { - "SCICAT_API_URL": "http://localhost:3000/", + "SCICAT_INGEST_URL": "http://localhost:3000/", "SCICAT_INGEST_USER": "test_user", "SCICAT_INGEST_PASSWORD": "test_password" }) @@ -150,8 +158,8 @@ def test_ingest_dataset_task(mocker: MockFixture): # Inject dummy ingestor module. dummy_ingestor = types.ModuleType("dummy_ingestor") dummy_ingestor.ingest = dummy_ingest - mocker.patch.dict(sys.modules, {"orchestration.flows.bl832.ingest_tomo832": dummy_ingestor}) + mocker.patch.dict(sys.modules, {"scicat_beamline_ingestion.ingesters.als_test_ingest": dummy_ingestor}) # Call the underlying function (.fn) of the task to bypass Prefect orchestration. - result = ingest_dataset_task.fn("dummy_file.h5", "orchestration.flows.bl832.ingest_tomo832") + result = scicat_ingest_flow.fn(dataset_path=Path("dummy_file.h5"), ingester_spec="bltest") assert result == "dummy_dataset_id" From 026fd9f7c42416f6688242aed3cd131bff413475 Mon Sep 17 00:00:00 2001 From: Garrett Birkel Date: Wed, 17 Dec 2025 17:00:30 -0800 Subject: [PATCH 09/15] This spec now lives in scicat_beamline_ingestion. --- orchestration/flows/bl832/move.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/orchestration/flows/bl832/move.py b/orchestration/flows/bl832/move.py index 9150c1d4..f72f0400 100644 --- a/orchestration/flows/bl832/move.py +++ b/orchestration/flows/bl832/move.py @@ -7,7 +7,7 @@ from prefect import flow, task, get_run_logger from prefect.variables import Variable -from orchestration.flows.scicat.ingest import ingest_dataset +from scicat_beamline_ingestion.flows import scicat_ingest_flow from orchestration.flows.bl832.config import Config832 from orchestration.globus.transfer import GlobusEndpoint, start_transfer from orchestration.prefect import schedule_prefect_flow @@ -15,7 +15,7 @@ API_KEY = os.getenv("API_KEY") -TOMO_INGESTOR_MODULE = "orchestration.flows.bl832.ingest_tomo832" +TOMO_INGESTOR_SPEC = "als_832_dx_4" @task(name="transfer_spot_to_data") @@ -150,9 +150,9 @@ def process_new_832_file_task( f"File successfully transferred from data832 to NERSC {file_path}. Task {task}" ) flow_name = f"ingest scicat: {Path(file_path).name}" - logger.info(f"Ingesting {file_path} with {TOMO_INGESTOR_MODULE}") + logger.info(f"Ingesting {file_path} with {TOMO_INGESTOR_SPEC}") try: - ingest_dataset(file_path, TOMO_INGESTOR_MODULE) + scicat_ingest_flow(dataset_path=Path(file_path), ingester_spec=TOMO_INGESTOR_SPEC) except Exception as e: logger.error(f"SciCat ingest failed with {e}") From 29ce108f95409f2ea4c9ac22ad72104374919863 Mon Sep 17 00:00:00 2001 From: Garrett Birkel Date: Wed, 17 Dec 2025 17:00:40 -0800 Subject: [PATCH 10/15] Not needed! --- orchestration/flows/scicat/ingest.py | 99 +--------------------------- 1 file changed, 2 insertions(+), 97 deletions(-) diff --git a/orchestration/flows/scicat/ingest.py b/orchestration/flows/scicat/ingest.py index 6c9cd543..ce610c8d 100644 --- a/orchestration/flows/scicat/ingest.py +++ b/orchestration/flows/scicat/ingest.py @@ -7,102 +7,7 @@ from orchestration.flows.scicat.utils import Issue - -@flow(name="scicat_dataset_ingest") -def ingest_dataset(file_path: str, ingestor: str): - """ Ingest a file into SciCat. - - Parameters - ---------- - file_path : str - Path where the file can be found on whatever server is processing this task - ingestor_module : str - Thy python module that contains the ingest function, e.g. "foo.bar.ingestor" - """ - ingest_dataset_task(file_path, ingestor) - - -@task(name="ingest_scicat") -def ingest_dataset_task(file_path: str, ingestor_module: str): - """ Ingest a file into SciCat. - - Parameters - ---------- - file_path : str - Path where the file can be found on whatever server is processing this task - ingestor_module : str - Thy python module that contains the ingest function, e.g. "foo.bar.ingestor" - """ - logger = get_run_logger() - - SCICAT_API_URL = os.getenv("SCICAT_API_URL") - SCICAT_INGEST_USER = os.getenv("SCICAT_INGEST_USER") - SCICAT_INGEST_PASSWORD = os.getenv("SCICAT_INGEST_PASSWORD") - - # files come in with the full pasth on the server that they - # were loaded from. - - # relative path: raw/... - # ingestor api maps /globa/cfs/cdirs/als/data_mover to /data_mover - # so we want to prepend /data_mover/8.3.2 - # if relative_path[0] == "/": - # relative_path = relative_path[1:] - # ingest_path = os.path.join("/data_mover/8.3.2", file_path) - logger.info( - f"Sending ingest job to {SCICAT_API_URL} for file {file_path}" - ) - try: - scicat_client = from_credentials( - SCICAT_API_URL, - SCICAT_INGEST_USER, - SCICAT_INGEST_PASSWORD) - except Exception as e: - logger.warning(f"Failed to create SciCat client using pyscicat method: {e}") - - # Note: the above method does not work with the current SciCat API (March 2025) - # The following method is used instead as a workaround, however, this will be udpated soon in pyscicat - # Ref: https://github.com/SciCatProject/pyscicat/pull/62 - try: - import requests - from urllib.parse import urljoin - - url = urljoin(SCICAT_API_URL, "auth/login") - logger.info(url) - response = requests.post( - url=url, - json={"username": SCICAT_INGEST_USER, "password": SCICAT_INGEST_PASSWORD}, - stream=False, - verify=True, - ) - # logger.info(f"Login response: {response.json()}") - scicat_client = ScicatClient(SCICAT_API_URL, response.json()["access_token"]) - logger.info("Logged in to SciCat.") - - except requests.exceptions.RequestException as e: - logger.error(f"Failed to log in to SciCat: {e}") - raise e - except Exception as e: - logger.error(f"Failed to log in to SciCat: {e}") - raise e - - ingestor_module = importlib.import_module(ingestor_module) - issues: List[Issue] = [] - new_dataset_id = ingestor_module.ingest( - scicat_client, - file_path, - issues, - ) - if len(issues) > 0: - logger.error(f"SciCat ingest failed with {len(issues)} issues") - for issue in issues: - logger.error(issue) - raise Exception("SciCat ingest failed") - return new_dataset_id - +# Stubbed; to be removed if __name__ == "__main__": - import sys - - from dotenv import load_dotenv - load_dotenv() - ingest_dataset(sys.argv[1], sys.argv[2]) + pass \ No newline at end of file From 20953009599689f2fe4c71a03f6d0b1e25738fd1 Mon Sep 17 00:00:00 2001 From: Garrett Birkel Date: Wed, 17 Dec 2025 17:00:57 -0800 Subject: [PATCH 11/15] Not needed, already in scicat_beamline_ingestion --- orchestration/flows/scicat/utils.py | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/orchestration/flows/scicat/utils.py b/orchestration/flows/scicat/utils.py index 7fc97012..1cc68710 100644 --- a/orchestration/flows/scicat/utils.py +++ b/orchestration/flows/scicat/utils.py @@ -69,26 +69,3 @@ def build_search_terms(sample_name): description = [term.lower() for term in terms if len(term) > 0] return " ".join(description) - -def encode_image_2_thumbnail(filebuffer, imType="jpg"): - logging.info("Creating thumbnail for dataset") - header = "data:image/{imType};base64,".format(imType=imType) - dataBytes = base64.b64encode(filebuffer.read()) - dataStr = dataBytes.decode("UTF-8") - return header + dataStr - - -def build_thumbnail(image_array: npt.ArrayLike): - image_array = image_array - np.min(image_array) + 1.001 - image_array = np.log(image_array) - image_array = 205 * image_array / (np.max(image_array)) - auto_contrast_image = Image.fromarray(image_array.astype("uint8")) - auto_contrast_image = ImageOps.autocontrast(auto_contrast_image, cutoff=0.1) - # filename = str(uuid4()) + ".png" - file = io.BytesIO() - # file = thumbnail_dir / Path(filename) - auto_contrast_image.save(file, format="png") - file.seek(0) - return file - - From b559f479a706f6ec1fbcc072304c73bb6c86aef1 Mon Sep 17 00:00:00 2001 From: Garrett Birkel Date: Thu, 18 Dec 2025 12:45:09 -0800 Subject: [PATCH 12/15] Renaming scicat-beamline-ingestion to scicat-beamline, and adding a Prefect Flow wrapper for it. --- orchestration/_tests/test_scicat.py | 6 ++-- orchestration/flows/bl832/move.py | 2 +- orchestration/flows/scicat/ingest.py | 51 ++++++++++++++++++++++++---- pyproject.toml | 2 +- requirements.txt | 2 +- 5 files changed, 50 insertions(+), 13 deletions(-) diff --git a/orchestration/_tests/test_scicat.py b/orchestration/_tests/test_scicat.py index 622066d7..471525e4 100644 --- a/orchestration/_tests/test_scicat.py +++ b/orchestration/_tests/test_scicat.py @@ -7,8 +7,8 @@ from pathlib import Path from pytest_mock import MockFixture -from scicat_beamline_ingestion.flows import scicat_ingest_flow -from orchestration.flows.scicat.utils import NPArrayEncoder, build_search_terms, calculate_access_controls, Issue +from orchestration.flows.scicat.utils import Issue +from orchestration.flows.scicat.ingest import scicat_ingest_flow from orchestration.flows.bl832.ingest_tomo832 import clean_email, UNKNOWN_EMAIL @@ -158,7 +158,7 @@ def test_ingest_dataset_task(mocker: MockFixture): # Inject dummy ingestor module. dummy_ingestor = types.ModuleType("dummy_ingestor") dummy_ingestor.ingest = dummy_ingest - mocker.patch.dict(sys.modules, {"scicat_beamline_ingestion.ingesters.als_test_ingest": dummy_ingestor}) + mocker.patch.dict(sys.modules, {"scicat_beamline.ingesters.als_test_ingest": dummy_ingestor}) # Call the underlying function (.fn) of the task to bypass Prefect orchestration. result = scicat_ingest_flow.fn(dataset_path=Path("dummy_file.h5"), ingester_spec="bltest") diff --git a/orchestration/flows/bl832/move.py b/orchestration/flows/bl832/move.py index f72f0400..38b28ec7 100644 --- a/orchestration/flows/bl832/move.py +++ b/orchestration/flows/bl832/move.py @@ -7,7 +7,7 @@ from prefect import flow, task, get_run_logger from prefect.variables import Variable -from scicat_beamline_ingestion.flows import scicat_ingest_flow +from orchestration.flows.scicat.ingest import scicat_ingest_flow from orchestration.flows.bl832.config import Config832 from orchestration.globus.transfer import GlobusEndpoint, start_transfer from orchestration.prefect import schedule_prefect_flow diff --git a/orchestration/flows/scicat/ingest.py b/orchestration/flows/scicat/ingest.py index ce610c8d..015d4ddf 100644 --- a/orchestration/flows/scicat/ingest.py +++ b/orchestration/flows/scicat/ingest.py @@ -1,13 +1,50 @@ -import importlib -import os -from typing import List +from typing import Optional, Dict, Any +from pathlib import Path -from pyscicat.client import ScicatClient, from_credentials -from prefect import flow, task, get_run_logger +from prefect import flow, get_run_logger -from orchestration.flows.scicat.utils import Issue -# Stubbed; to be removed +from scicat_beamline import ingest + + +@flow(name="scicat-ingest-flow") +def scicat_ingest_flow( + dataset_path: Path, + ingester_spec: Optional[str], + owner_username: Optional[str], + scicat_url: Optional[str], + scicat_username: Optional[str], + scicat_password: Optional[str], +) -> Dict[str, Any]: + """ + Runs the SciCat ingestion process implemented for the given spec identifier, + on the given folder or file. + Args: + dataset_path: Path of the asset to ingest. May be file or directory depending on the spec. + If SICAT_INGEST_INTERNAL_BASE_FOLDER or SCICAT_INGEST_BASE_FOLDER is set, this path is + considered relative to that base folder. + These remaining args are optional; if not provided, environment variables will be used. + ingester_spec: Spec to ingest with. (or set SCICAT_INGEST_INGESTER_SPEC) + owner_username: User doing the ingesting. May be different from the user_name, especially if using a token (or set SCICAT_INGEST_OWNER_USERNAME) + scicat_url: Scicat server base url. If not provided, will try localhost default (or set SCICAT_INGEST_URL) + scicat_username: Scicat server username (or set SCICAT_INGEST_USERNAME) + scicat_password: Scicat server password (or set SCICAT_INGEST_PASSWORD) + Returns: + Dict containing task results or skip message + """ + # Get the Prefect logger for the current flow run + prefect_adapter = get_run_logger() + + return ingest( + dataset_path=dataset_path, + ingester_spec=ingester_spec, + owner_username=owner_username, + scicat_url=scicat_url, + scicat_username=scicat_username, + scicat_password=scicat_password, + logger=prefect_adapter.logger + ) + if __name__ == "__main__": pass \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 725f73cb..c121411f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,7 +21,7 @@ dependencies = [ "pydantic==2.11", "python-dotenv", "pyyaml", - "scicat_beamline_ingestion @ git+https://github.com/als-computing/scicat_beamline_ingestion.git@main", + "scicat_beamline @ git+https://github.com/als-computing/scicat_beamline.git@main", "sfapi_client" ] diff --git a/requirements.txt b/requirements.txt index 17449efb..3ed219ca 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,5 +14,5 @@ prometheus_client==0.21.1 pydantic==2.11 python-dotenv pyyaml -scicat-beamline-ingestion @ git+https://github.com/als-computing/scicat_beamline_ingestion.git@main +scicat-beamline @ git+https://github.com/als-computing/scicat_beamline.git@main sfapi_client From ba9fcf0c04d276c41050e5718bd3ebcd1184acb5 Mon Sep 17 00:00:00 2001 From: Garrett Birkel Date: Thu, 18 Dec 2025 12:45:25 -0800 Subject: [PATCH 13/15] Taking out some tests that were moved to scicat_beamline --- orchestration/_tests/test_scicat.py | 103 ++-------------------------- orchestration/flows/scicat/utils.py | 42 ------------ 2 files changed, 6 insertions(+), 139 deletions(-) diff --git a/orchestration/_tests/test_scicat.py b/orchestration/_tests/test_scicat.py index 471525e4..21b4e7e4 100644 --- a/orchestration/_tests/test_scicat.py +++ b/orchestration/_tests/test_scicat.py @@ -10,104 +10,13 @@ from orchestration.flows.scicat.utils import Issue from orchestration.flows.scicat.ingest import scicat_ingest_flow -from orchestration.flows.bl832.ingest_tomo832 import clean_email, UNKNOWN_EMAIL - - -# TODO: Move these to scicat_beamline_ingestion -def test_clean_email_valid(): - # Remove surrounding whitespace. - assert clean_email(" user@example.com ") == "user@example.com" - - -def test_clean_email_none(): - # Non-string input returns the default unknown email. - assert clean_email(None) == UNKNOWN_EMAIL - - -def test_clean_email_empty(): - # An empty string (or only spaces) should return the default. - assert clean_email(" ") == UNKNOWN_EMAIL - - -def test_clean_email_no_at_symbol(): - # A string without an "@" should return the default. - assert clean_email("invalid-email") == UNKNOWN_EMAIL - - -def test_clean_email_literal_none(): - # The string "NONE" (case-insensitive) should return the default. - assert clean_email("NONE") == UNKNOWN_EMAIL - assert clean_email("none") == UNKNOWN_EMAIL - - -def test_clean_email_internal_spaces(): - # Spaces inside the email should be removed. - # For example, " user @ example.com " should be cleaned to "user@example.com" - assert clean_email(" user @ example.com ") == "user@example.com" - - -def add_mock_requests(mock_request): - mock_request.post("http://localhost:3000/api/v3/Users/login", json={"id": "foobar"}) - mock_request.post("http://localhost:3000/api/v3/Samples", json={"sampleId": "dataset_id"}) - mock_request.post("http://localhost:3000/api/v3/RawDatasets/replaceOrCreate", json={"pid": "42"}) - mock_request.post("http://localhost:3000/api/v3/RawDatasets/42/origdatablocks", json={"response": "random"}) - - -def test_np_encoder(): - test_dict = {"dont_panic": np.array([1, 2, 3], dtype=np.int8)} - assert json.dumps(test_dict, cls=NPArrayEncoder) - - test_dict = {"dont_panic": np.array([1, 2, 3], dtype=np.float32)} - assert json.dumps(test_dict, cls=NPArrayEncoder) - - test_dict = {"dont_panic": np.full((1, 1), np.inf)} - encoded_np = json.loads(json.dumps(test_dict, cls=NPArrayEncoder)) - assert json.dumps(encoded_np, allow_nan=False) - - -def test_build_search_terms(): - terms = build_search_terms("Time-is_an illusion. Lunchtime/2x\\so.") - assert "time" in terms - assert "is" in terms - assert "an" in terms - assert "illusion" in terms - assert "lunchtime" in terms - assert "2x" in terms - assert "so" in terms - - -def test_access_controls(): - username = "slartibartfast" - # no proposal, no beamline - access_controls = calculate_access_controls(username, None, None) - assert access_controls["owner_group"] == "slartibartfast" - assert access_controls["access_groups"] == [] - - # proposal and no beamline - access_controls = calculate_access_controls(username, None, "42") - assert access_controls["owner_group"] == "42" - - # no proposal and beamline - access_controls = calculate_access_controls(username, "10.3.1", None) - assert access_controls["owner_group"] == "slartibartfast" - assert "10.3.1" in access_controls["access_groups"] - assert "slartibartfast" in access_controls["access_groups"] - - # proposal and beamline - access_controls = calculate_access_controls(username, "10.3.1", "42") - assert access_controls["owner_group"] == "42" - assert "10.3.1" in access_controls["access_groups"] - - # special 8.3.2 mapping - access_controls = calculate_access_controls(username, "bl832", "42") - assert access_controls["owner_group"] == "42" - assert "8.3.2" in access_controls["access_groups"] - assert "bl832" in access_controls["access_groups"] - - -def test_clean_email(): - pass +# This code is in an indeterminate state because it is unclear whether +# the SciCat ingest flow should be tested here, when it is already tested +# with Prefect in the scicat_beamline package tests. +# There may be some need for end-to-end testing where the destination SciCat +# server is mocked or switched to the staging server, and in that case this code +# may be useful. # Dummy functions and modules for external dependencies. def dummy_requests_post(*args, **kwargs): diff --git a/orchestration/flows/scicat/utils.py b/orchestration/flows/scicat/utils.py index 1cc68710..2e657412 100644 --- a/orchestration/flows/scicat/utils.py +++ b/orchestration/flows/scicat/utils.py @@ -27,45 +27,3 @@ class Issue: severity: Severity msg: str exception: Optional[Union[str, None]] = None - - -class NPArrayEncoder(json.JSONEncoder): - def default(self, obj): - if isinstance(obj, np.integer): - return int(obj) - if isinstance(obj, np.floating): - return float(obj) - if isinstance(obj, np.ndarray): - return [None if np.isnan(item) or np.isinf(item) else item for item in obj] - return json.JSONEncoder.default(self, obj) - - -def calculate_access_controls(username, beamline, proposal) -> Dict: - # make an access group list that includes the name of the proposal and the name of the beamline - access_groups = [] - # sometimes the beamline name is super dirty " '8.3.2', "" '8.3.2', " - beamline = beamline.replace(" '", "").replace("', ", "") if beamline else None - # set owner_group to username so that at least someone has access in case no proposal number is found - owner_group = username - if beamline: - access_groups.append(beamline) - # username lets the user see the Dataset in order to ingest objects after the Dataset - access_groups.append(username) - # temporary mapping while beamline controls process request to match beamline name with what comes - # from ALSHub - if beamline == "bl832" and "8.3.2" not in access_groups: - access_groups.append("8.3.2") - - if proposal and proposal != "None": - owner_group = proposal - - # this is a bit of a kludge. Add 8.3.2 into the access groups so that staff will be able to see it - return {"owner_group": owner_group, "access_groups": access_groups} - - -def build_search_terms(sample_name): - """extract search terms from sample name to provide something pleasing to search on""" - terms = re.split("[^a-zA-Z0-9]", sample_name) - description = [term.lower() for term in terms if len(term) > 0] - return " ".join(description) - From 36270d5c13f9a35c0d662703518da43bf723867d Mon Sep 17 00:00:00 2001 From: Garrett Birkel Date: Thu, 18 Dec 2025 14:56:51 -0800 Subject: [PATCH 14/15] Disabling this for now. --- orchestration/_tests/test_scicat.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/orchestration/_tests/test_scicat.py b/orchestration/_tests/test_scicat.py index 21b4e7e4..961fdef3 100644 --- a/orchestration/_tests/test_scicat.py +++ b/orchestration/_tests/test_scicat.py @@ -48,6 +48,9 @@ def dummy_ingest( def test_ingest_dataset_task(mocker: MockFixture): + + return # Temporarily disable the test. + # Set environment variables. mocker.patch.dict(os.environ, { "SCICAT_INGEST_URL": "http://localhost:3000/", From 68155212251027e7c6a4fb58bf6ee653807ede98 Mon Sep 17 00:00:00 2001 From: Garrett Birkel Date: Fri, 19 Dec 2025 13:52:36 -0800 Subject: [PATCH 15/15] All of these are truly optional! --- orchestration/flows/scicat/ingest.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/orchestration/flows/scicat/ingest.py b/orchestration/flows/scicat/ingest.py index 015d4ddf..38e31413 100644 --- a/orchestration/flows/scicat/ingest.py +++ b/orchestration/flows/scicat/ingest.py @@ -10,11 +10,11 @@ @flow(name="scicat-ingest-flow") def scicat_ingest_flow( dataset_path: Path, - ingester_spec: Optional[str], - owner_username: Optional[str], - scicat_url: Optional[str], - scicat_username: Optional[str], - scicat_password: Optional[str], + ingester_spec: str | None = None, + owner_username: str | None = None, + scicat_url: str | None = None, + scicat_username: str | None = None, + scicat_password: str | None = None, ) -> Dict[str, Any]: """ Runs the SciCat ingestion process implemented for the given spec identifier,