From b8666fe8dcba9dbd252f361ee73e1afe75dfb21b Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Wed, 3 Jun 2026 19:54:33 -0400 Subject: [PATCH 01/16] Sync up unique cache paths for Singularity images with cwltool --- src/cwl_utils/image_puller.py | 14 ++++++---- src/cwl_utils/tests/test_image_puller.py | 34 ++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 5 deletions(-) create mode 100644 src/cwl_utils/tests/test_image_puller.py diff --git a/src/cwl_utils/image_puller.py b/src/cwl_utils/image_puller.py index 5604002d..b42fff0c 100644 --- a/src/cwl_utils/image_puller.py +++ b/src/cwl_utils/image_puller.py @@ -22,7 +22,11 @@ def __init__( cmd: str, force_pull: bool, ) -> None: - """Create an ImagePuller.""" + """ + Create an ImagePuller. + + req already contains any tag that will be used. + """ self.req = req self.save_directory = save_directory self.cmd = cmd @@ -86,14 +90,14 @@ def save_docker_image(self) -> None: class SingularityImagePuller(ImagePuller): """Pull docker image with Singularity.""" - CHARS_TO_REPLACE = ["/", ":"] - NEW_CHAR = "_" + CHARS_TO_REPLACE = ["_", "/"] + NEW_STRINGS = ["___", "_s_"] def get_image_name(self) -> str: """Determine the file name appropriate to the installed version of Singularity.""" image_name = self.req - for char in self.CHARS_TO_REPLACE: - image_name = image_name.replace(char, self.NEW_CHAR) + for char, replacement in zip(self.CHARS_TO_REPLACE, self.NEW_STRINGS): + image_name = image_name.replace(char, replacement) if is_singularity_version_2_6(): suffix = ".img" elif is_singularity_version_3_or_newer(): diff --git a/src/cwl_utils/tests/test_image_puller.py b/src/cwl_utils/tests/test_image_puller.py new file mode 100644 index 00000000..cc474c20 --- /dev/null +++ b/src/cwl_utils/tests/test_image_puller.py @@ -0,0 +1,34 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Tests for classes for docker-extract.""" + +from cwl_utils.image_puller import SingularityImagePuller +from cwl_utils.singularity import get_version as get_singularity_version +from cwl_utils.singularity import is_version_2_6 as is_singularity_version_2_6 +from cwl_utils.singularity import ( + is_version_3_or_newer as is_singularity_version_3_or_newer, +) + +from .util import needs_singularity + + +@needs_singularity +class TestSingularityImagePuller: + """Tests for SingularityImagePuller.""" + + def test_get_image_name_matches_cwltool(self) -> None: + """Make sure image names generated match those expected by cwltool.""" + + if is_singularity_version_2_6(): + suffix = ".img" + elif is_singularity_version_3_or_newer(): + suffix = ".sif" + else: + raise Exception( + f"Don't know how to handle this version of singularity: {get_singularity_version()}." + ) + + def get_name(s: str) -> str: + return SingularityImagePuller(s, None, "", False).get_image_name() + + assert get_name("some_name/repo:123") == f"some___name_s_repo:123{suffix}" + assert get_name("some/name_repo:123") == f"some_s_name___repo:123{suffix}" From e980f78197cf551fc0e9cbdb3d7b989b01433a03 Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Wed, 3 Jun 2026 20:05:07 -0400 Subject: [PATCH 02/16] Remove blank lines not allowed after docstrings --- src/cwl_utils/tests/test_image_puller.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/cwl_utils/tests/test_image_puller.py b/src/cwl_utils/tests/test_image_puller.py index cc474c20..58006fcb 100644 --- a/src/cwl_utils/tests/test_image_puller.py +++ b/src/cwl_utils/tests/test_image_puller.py @@ -17,7 +17,6 @@ class TestSingularityImagePuller: def test_get_image_name_matches_cwltool(self) -> None: """Make sure image names generated match those expected by cwltool.""" - if is_singularity_version_2_6(): suffix = ".img" elif is_singularity_version_3_or_newer(): From 1121d08d431c1eb695cda4e742979221ce067f2b Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Thu, 23 Jul 2026 17:32:19 -0400 Subject: [PATCH 03/16] Try a versioned cache with lookup of old images under old schemes --- src/cwl_utils/image_puller.py | 71 +++++++++++++++++++++++++++++++---- 1 file changed, 64 insertions(+), 7 deletions(-) diff --git a/src/cwl_utils/image_puller.py b/src/cwl_utils/image_puller.py index b42fff0c..b26ed131 100644 --- a/src/cwl_utils/image_puller.py +++ b/src/cwl_utils/image_puller.py @@ -92,10 +92,27 @@ class SingularityImagePuller(ImagePuller): CHARS_TO_REPLACE = ["_", "/"] NEW_STRINGS = ["___", "_s_"] + # This ends up being a directory name that often gets dropped into the user's current directory + FILENAME_SCHEME_VERSION = "v2" + + def _image_to_filename( + self, + image_name: str, + to_replace: list[str], + replacements: list[str], + version: str | None = None + ) -> str: + """ + Get the filename for an image, using the given replacements to escape it. - def get_image_name(self) -> str: - """Determine the file name appropriate to the installed version of Singularity.""" - image_name = self.req + The filename will be appropriate for the current Singularity. + + The filename will include a disambiguating version if version is set. + No filenames from two different versions, or with and without a + version, will be equal. + + The filename may contain leading directory components. + """ for char, replacement in zip(self.CHARS_TO_REPLACE, self.NEW_STRINGS): image_name = image_name.replace(char, replacement) if is_singularity_version_2_6(): @@ -106,7 +123,38 @@ def get_image_name(self) -> str: raise Exception( f"Don't know how to handle this version of singularity: {get_singularity_version()}." ) - return f"{image_name}{suffix}" + filename = f"{image_name}{suffix}" + if version is not None: + filename = os.path.join(version, filename) + return filename + + def get_image_name(self) -> str: + """Determine the file name appropriate to the installed version of Singularity.""" + image_name = self.req + return self._image_to_filename(image_name, CHARS_TO_REPLACE, NEW_STRINGS, FILENAME_SCHEME_VERSION) + + def get_alternate_image_names(self) -> list[str]: + """ + Determine filenames used by previous versions of cwltool or cwl-utils. + + These should be checked for the image and used if it exists there, + instead of pulling it again. + + These cover cwltool 3.2.20260720092025 and cwl-utils 0.42. + """ + image_name = self.req + return [ + # Check the path cwl-utils 0.42 uses, with underscores for slashes + # and colons. + self._image_to_filename(image_name, ["/", ":"], ["_", "_"]), + # Check the path cwltool 3.2.20260720092025 uses, with _latest + # potentially appended and then only slashes replaced. + self._image_to_filename( + image_name + "_latest" if ":" not in image_name else image_name, + ["/"], + ["_"], + ), + ] def save_docker_image(self) -> None: """Pull down the Docker software container image and save it in the Singularity image format.""" @@ -114,10 +162,19 @@ def save_docker_image(self) -> None: if self.save_directory: save_directory = self.save_directory target = Path(save_directory, self.get_image_name()) - if target.exists() and not self.force_pull: - _LOGGER.info(f"Already cached {self.req} with Singularity.") - return + if not self.force_pull: + if target.exists(): + _LOGGER.info(f"Already cached {self.req} with Singularity.") + return + # Otherwise check other paths old versions may have placed it at. + alternate_targets = [Path(save_directory, img) for img in self.get_alternate_image_names()] + for alt_target in alternate_targets: + if alt_target.exists(): + _LOGGER.info(f"Already cached {self.req} with Singularity using a previous caching scheme.") + return + _LOGGER.info(f"Pulling {self.req} with Singularity...") + os.makedirs(target.parent, exist_ok=True) cmd_pull = [ self.cmd, "pull", From 1481421c49e7a2a9182e1eac0651f9bd128caab4 Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Thu, 23 Jul 2026 17:50:10 -0400 Subject: [PATCH 04/16] Use a part of the filename space we won't take as old image paths, instead of versioning --- src/cwl_utils/image_puller.py | 47 ++++++++++++++++++++++++----------- 1 file changed, 32 insertions(+), 15 deletions(-) diff --git a/src/cwl_utils/image_puller.py b/src/cwl_utils/image_puller.py index b26ed131..e9894405 100644 --- a/src/cwl_utils/image_puller.py +++ b/src/cwl_utils/image_puller.py @@ -92,26 +92,17 @@ class SingularityImagePuller(ImagePuller): CHARS_TO_REPLACE = ["_", "/"] NEW_STRINGS = ["___", "_s_"] - # This ends up being a directory name that often gets dropped into the user's current directory - FILENAME_SCHEME_VERSION = "v2" def _image_to_filename( self, image_name: str, to_replace: list[str], replacements: list[str], - version: str | None = None ) -> str: """ Get the filename for an image, using the given replacements to escape it. The filename will be appropriate for the current Singularity. - - The filename will include a disambiguating version if version is set. - No filenames from two different versions, or with and without a - version, will be equal. - - The filename may contain leading directory components. """ for char, replacement in zip(self.CHARS_TO_REPLACE, self.NEW_STRINGS): image_name = image_name.replace(char, replacement) @@ -124,14 +115,35 @@ def _image_to_filename( f"Don't know how to handle this version of singularity: {get_singularity_version()}." ) filename = f"{image_name}{suffix}" - if version is not None: - filename = os.path.join(version, filename) return filename + def _could_be_current_image(filename: str) -> bool: + """ + Check if a path could belong to the current image name encoding scheme. + + This allows us to be backward-compatible with most existing cached + images, without risking treating cache entries created under the new + scheme as belonging to different images under older schemes. + + Is not guaranteed to be a tight bound: may return True for things that + can't actually be generated under the new scheme, but will never + return False for things that can. + """ + for replacement in NEW_STRINGS: + # Remove anything the new scheme generates involving replaceable + # characters. + filename = filename.replace(replacement, "") + for remaining in CHARS_TO_REPLACE: + if remaining in filename: + # We have something that can't have been generated under the + # new scheme. + return False + # If we don't see anything we can't make, we can probably make this path. + return True + def get_image_name(self) -> str: """Determine the file name appropriate to the installed version of Singularity.""" - image_name = self.req - return self._image_to_filename(image_name, CHARS_TO_REPLACE, NEW_STRINGS, FILENAME_SCHEME_VERSION) + return self._image_to_filename(self.req, CHARS_TO_REPLACE, NEW_STRINGS) def get_alternate_image_names(self) -> list[str]: """ @@ -141,9 +153,12 @@ def get_alternate_image_names(self) -> list[str]: instead of pulling it again. These cover cwltool 3.2.20260720092025 and cwl-utils 0.42. + + If an image name could potentially also belong to some image under the + current scheme, it will not appear here. """ image_name = self.req - return [ + possibilities = [ # Check the path cwl-utils 0.42 uses, with underscores for slashes # and colons. self._image_to_filename(image_name, ["/", ":"], ["_", "_"]), @@ -155,6 +170,8 @@ def get_alternate_image_names(self) -> list[str]: ["_"], ), ] + possibilities = [p for p in possibilities if not self._could_be_current_image(p)] + return possibilities def save_docker_image(self) -> None: """Pull down the Docker software container image and save it in the Singularity image format.""" @@ -190,5 +207,5 @@ def save_docker_image(self) -> None: ) ImagePuller._run_command_pull(cmd_pull) _LOGGER.info( - f"Image successfully pulled: {save_directory}/{self.get_image_name()}" + f"Image successfully pulled: {target}" ) From ab6ed6e557bd88439040d32556a246583111fa2b Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Thu, 23 Jul 2026 17:54:24 -0400 Subject: [PATCH 05/16] Pass linting --- src/cwl_utils/image_puller.py | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/src/cwl_utils/image_puller.py b/src/cwl_utils/image_puller.py index e9894405..cb8131fb 100644 --- a/src/cwl_utils/image_puller.py +++ b/src/cwl_utils/image_puller.py @@ -92,7 +92,7 @@ class SingularityImagePuller(ImagePuller): CHARS_TO_REPLACE = ["_", "/"] NEW_STRINGS = ["___", "_s_"] - + def _image_to_filename( self, image_name: str, @@ -117,7 +117,7 @@ def _image_to_filename( filename = f"{image_name}{suffix}" return filename - def _could_be_current_image(filename: str) -> bool: + def _could_be_current_image(self, filename: str) -> bool: """ Check if a path could belong to the current image name encoding scheme. @@ -129,11 +129,11 @@ def _could_be_current_image(filename: str) -> bool: can't actually be generated under the new scheme, but will never return False for things that can. """ - for replacement in NEW_STRINGS: + for replacement in self.NEW_STRINGS: # Remove anything the new scheme generates involving replaceable # characters. filename = filename.replace(replacement, "") - for remaining in CHARS_TO_REPLACE: + for remaining in self.CHARS_TO_REPLACE: if remaining in filename: # We have something that can't have been generated under the # new scheme. @@ -143,7 +143,9 @@ def _could_be_current_image(filename: str) -> bool: def get_image_name(self) -> str: """Determine the file name appropriate to the installed version of Singularity.""" - return self._image_to_filename(self.req, CHARS_TO_REPLACE, NEW_STRINGS) + return self._image_to_filename( + self.req, self.CHARS_TO_REPLACE, self.NEW_STRINGS + ) def get_alternate_image_names(self) -> list[str]: """ @@ -170,7 +172,9 @@ def get_alternate_image_names(self) -> list[str]: ["_"], ), ] - possibilities = [p for p in possibilities if not self._could_be_current_image(p)] + possibilities = [ + p for p in possibilities if not self._could_be_current_image(p) + ] return possibilities def save_docker_image(self) -> None: @@ -184,14 +188,17 @@ def save_docker_image(self) -> None: _LOGGER.info(f"Already cached {self.req} with Singularity.") return # Otherwise check other paths old versions may have placed it at. - alternate_targets = [Path(save_directory, img) for img in self.get_alternate_image_names()] + alternate_targets = [ + Path(save_directory, img) for img in self.get_alternate_image_names() + ] for alt_target in alternate_targets: if alt_target.exists(): - _LOGGER.info(f"Already cached {self.req} with Singularity using a previous caching scheme.") + _LOGGER.info( + f"Already cached {self.req} with Singularity using a previous caching scheme." + ) return - + _LOGGER.info(f"Pulling {self.req} with Singularity...") - os.makedirs(target.parent, exist_ok=True) cmd_pull = [ self.cmd, "pull", @@ -206,6 +213,4 @@ def save_docker_image(self) -> None: ] ) ImagePuller._run_command_pull(cmd_pull) - _LOGGER.info( - f"Image successfully pulled: {target}" - ) + _LOGGER.info(f"Image successfully pulled: {target}") From 5cb41b4491e6e43e7d228bd1aa1669cacacdb30c Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Thu, 23 Jul 2026 18:09:51 -0400 Subject: [PATCH 06/16] Add tests for backward-compatible name generation --- src/cwl_utils/tests/test_image_puller.py | 44 ++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/src/cwl_utils/tests/test_image_puller.py b/src/cwl_utils/tests/test_image_puller.py index 58006fcb..e243312a 100644 --- a/src/cwl_utils/tests/test_image_puller.py +++ b/src/cwl_utils/tests/test_image_puller.py @@ -31,3 +31,47 @@ def get_name(s: str) -> str: assert get_name("some_name/repo:123") == f"some___name_s_repo:123{suffix}" assert get_name("some/name_repo:123") == f"some_s_name___repo:123{suffix}" + + def test_get_image_alternate_names_match_old_cwltool(self) -> None: + """ + Make sure alternate image names tries match those previously used by + cwltool 3.2.20260720092025. + """ + if is_singularity_version_2_6(): + suffix = ".img" + elif is_singularity_version_3_or_newer(): + suffix = ".sif" + else: + raise Exception( + f"Don't know how to handle this version of singularity: {get_singularity_version()}." + ) + + def get_names(s: str) -> list[str]: + return SingularityImagePuller( + s, None, "", False + ).get_alternate_image_names() + + assert f"debian:stable-slim{suffix}" in get_names("debian:stable-slim") + assert f"quay.io_user_image_latest{suffix}" in get_names("quay.io/user/image") + + def test_get_image_alternate_names_match_old_cwl_utils(self) -> None: + """ + Make sure alternate image names tries match those previously used by + cwl-utils 0.42 + """ + if is_singularity_version_2_6(): + suffix = ".img" + elif is_singularity_version_3_or_newer(): + suffix = ".sif" + else: + raise Exception( + f"Don't know how to handle this version of singularity: {get_singularity_version()}." + ) + + def get_names(s: str) -> list[str]: + return SingularityImagePuller( + s, None, "", False + ).get_alternate_image_names() + + assert f"debian_stable-slim{suffix}" in get_names("debian:stable-slim") + assert f"quay.io_user_image{suffix}" in get_names("quay.io/user/image") From a2da4a0ea54778be35bcb333f0c187f266540ba6 Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Thu, 23 Jul 2026 18:50:45 -0400 Subject: [PATCH 07/16] Fix typo --- src/cwl_utils/tests/test_image_puller.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/cwl_utils/tests/test_image_puller.py b/src/cwl_utils/tests/test_image_puller.py index e243312a..1226d253 100644 --- a/src/cwl_utils/tests/test_image_puller.py +++ b/src/cwl_utils/tests/test_image_puller.py @@ -34,7 +34,7 @@ def get_name(s: str) -> str: def test_get_image_alternate_names_match_old_cwltool(self) -> None: """ - Make sure alternate image names tries match those previously used by + Make sure alternate image names tried match those previously used by cwltool 3.2.20260720092025. """ if is_singularity_version_2_6(): @@ -56,7 +56,7 @@ def get_names(s: str) -> list[str]: def test_get_image_alternate_names_match_old_cwl_utils(self) -> None: """ - Make sure alternate image names tries match those previously used by + Make sure alternate image names tried match those previously used by cwl-utils 0.42 """ if is_singularity_version_2_6(): From a01eeddea265020307ec29a64b2130afeb3931a4 Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Thu, 23 Jul 2026 19:07:47 -0400 Subject: [PATCH 08/16] Allow the matching name for the old name to be the primary name --- src/cwl_utils/tests/test_image_puller.py | 26 +++++++++++++----------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/src/cwl_utils/tests/test_image_puller.py b/src/cwl_utils/tests/test_image_puller.py index 1226d253..4249493d 100644 --- a/src/cwl_utils/tests/test_image_puller.py +++ b/src/cwl_utils/tests/test_image_puller.py @@ -32,10 +32,14 @@ def get_name(s: str) -> str: assert get_name("some_name/repo:123") == f"some___name_s_repo:123{suffix}" assert get_name("some/name_repo:123") == f"some_s_name___repo:123{suffix}" - def test_get_image_alternate_names_match_old_cwltool(self) -> None: + # We have to include the normal name here because if there aren't + # slashes or underscores and a tag is included we generate the same + # names for the same images under the new and old schemes. + + def test_get_image_names_match_old_cwltool(self) -> None: """ - Make sure alternate image names tried match those previously used by - cwltool 3.2.20260720092025. + Make sure main and alternate image names tried include those previously + used by cwltool 3.2.20260720092025. """ if is_singularity_version_2_6(): suffix = ".img" @@ -47,17 +51,16 @@ def test_get_image_alternate_names_match_old_cwltool(self) -> None: ) def get_names(s: str) -> list[str]: - return SingularityImagePuller( - s, None, "", False - ).get_alternate_image_names() + puller = SingularityImagePuller(s, None, "", False) + return [puller.get_image_name()] + puller.get_alternate_image_names() assert f"debian:stable-slim{suffix}" in get_names("debian:stable-slim") assert f"quay.io_user_image_latest{suffix}" in get_names("quay.io/user/image") - def test_get_image_alternate_names_match_old_cwl_utils(self) -> None: + def test_get_image_names_match_old_cwl_utils(self) -> None: """ - Make sure alternate image names tried match those previously used by - cwl-utils 0.42 + Make sure main and alternate image names tried include those previously + used by cwl-utils 0.42 """ if is_singularity_version_2_6(): suffix = ".img" @@ -69,9 +72,8 @@ def test_get_image_alternate_names_match_old_cwl_utils(self) -> None: ) def get_names(s: str) -> list[str]: - return SingularityImagePuller( - s, None, "", False - ).get_alternate_image_names() + puller = SingularityImagePuller(s, None, "", False) + return [puller.get_image_name()] + puller.get_alternate_image_names() assert f"debian_stable-slim{suffix}" in get_names("debian:stable-slim") assert f"quay.io_user_image{suffix}" in get_names("quay.io/user/image") From 5486625aef1961300b6a22a712e4c6e7431bcb29 Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Thu, 23 Jul 2026 19:17:28 -0400 Subject: [PATCH 09/16] Invent accepted summaries --- src/cwl_utils/tests/test_image_puller.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/cwl_utils/tests/test_image_puller.py b/src/cwl_utils/tests/test_image_puller.py index 4249493d..122c1f9f 100644 --- a/src/cwl_utils/tests/test_image_puller.py +++ b/src/cwl_utils/tests/test_image_puller.py @@ -38,6 +38,8 @@ def get_name(s: str) -> str: def test_get_image_names_match_old_cwltool(self) -> None: """ + Check image names against cwltool. + Make sure main and alternate image names tried include those previously used by cwltool 3.2.20260720092025. """ @@ -59,6 +61,8 @@ def get_names(s: str) -> list[str]: def test_get_image_names_match_old_cwl_utils(self) -> None: """ + Check image names against cwl-utils. + Make sure main and alternate image names tried include those previously used by cwl-utils 0.42 """ From 51f4fabf6d862c28707e405db60f03fd602d53e8 Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Thu, 23 Jul 2026 19:30:52 -0400 Subject: [PATCH 10/16] Add possibly useless parentheses in case that somehow fixes alternate name generation --- src/cwl_utils/image_puller.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cwl_utils/image_puller.py b/src/cwl_utils/image_puller.py index cb8131fb..c93dc7b7 100644 --- a/src/cwl_utils/image_puller.py +++ b/src/cwl_utils/image_puller.py @@ -167,7 +167,7 @@ def get_alternate_image_names(self) -> list[str]: # Check the path cwltool 3.2.20260720092025 uses, with _latest # potentially appended and then only slashes replaced. self._image_to_filename( - image_name + "_latest" if ":" not in image_name else image_name, + (image_name + "_latest") if ":" not in image_name else image_name, ["/"], ["_"], ), From faf2c3ece6f7d672d07dc7c89e997666b6266aca Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Thu, 23 Jul 2026 17:21:04 -0700 Subject: [PATCH 11/16] Make _image_to_filename() stop ignoring its arguments --- src/cwl_utils/image_puller.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cwl_utils/image_puller.py b/src/cwl_utils/image_puller.py index c93dc7b7..d3c61e20 100644 --- a/src/cwl_utils/image_puller.py +++ b/src/cwl_utils/image_puller.py @@ -104,7 +104,7 @@ def _image_to_filename( The filename will be appropriate for the current Singularity. """ - for char, replacement in zip(self.CHARS_TO_REPLACE, self.NEW_STRINGS): + for char, replacement in zip(to_replace, replacements): image_name = image_name.replace(char, replacement) if is_singularity_version_2_6(): suffix = ".img" From fd5735648d3577253fc6bd30dece0bd7b522e84c Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Thu, 30 Jul 2026 12:33:29 -0400 Subject: [PATCH 12/16] Avoid failing codecov upload due to missing GPG key --- .github/workflows/ci-tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 29b4adbc..c37211e3 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -75,6 +75,8 @@ jobs: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: fail_ci_if_error: true + # Work around https://github.com/codecov/codecov-action/issues/1876 + use_pypi: true tox-style: name: CI linters via Tox From 4b1a2efd354f1f03288195fa3c67e511801ae7f0 Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Thu, 30 Jul 2026 12:50:05 -0400 Subject: [PATCH 13/16] Make tests rerun From 4844fe18820183675e2ed727efcd9ed76c03c2ff Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Tue, 4 Aug 2026 14:33:26 -0700 Subject: [PATCH 14/16] Expose singularity-pulling machinery cwltool needs --- src/cwl_utils/image_puller.py | 149 +++++++++++++++++++++++++++++----- 1 file changed, 127 insertions(+), 22 deletions(-) diff --git a/src/cwl_utils/image_puller.py b/src/cwl_utils/image_puller.py index d3c61e20..b2f735da 100644 --- a/src/cwl_utils/image_puller.py +++ b/src/cwl_utils/image_puller.py @@ -2,9 +2,12 @@ """Classes for docker-extract.""" import logging +import os +import shutil import subprocess # nosec from abc import ABC, abstractmethod from pathlib import Path +from uuid import uuid4 from .singularity import get_version as get_singularity_version from .singularity import is_version_2_6 as is_singularity_version_2_6 @@ -41,10 +44,17 @@ def save_docker_image(self) -> None: """Download and save the image to disk.""" @staticmethod - def _run_command_pull(cmd_pull: list[str]) -> None: + def _run_command_pull( + cmd_pull: list[str], + env_pull: dict[str, str] | None = None, + ) -> None: try: subprocess.run( # nosec - cmd_pull, check=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT + cmd_pull, + env=env_pull, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, ) except subprocess.CalledProcessError as err: if err.output: @@ -88,7 +98,14 @@ def save_docker_image(self) -> None: class SingularityImagePuller(ImagePuller): - """Pull docker image with Singularity.""" + """ + Pull docker image with Singularity. + + The image req may not contain a protocol. + + The image req, if it refers to a Docker image, may or may not contain a + tag. + """ CHARS_TO_REPLACE = ["_", "/"] NEW_STRINGS = ["___", "_s_"] @@ -177,40 +194,128 @@ def get_alternate_image_names(self) -> list[str]: ] return possibilities - def save_docker_image(self) -> None: - """Pull down the Docker software container image and save it in the Singularity image format.""" + def find_destination_path(self) -> Path: + """ + Find the path where the image belongs. + """ save_directory: str | Path if self.save_directory: save_directory = self.save_directory - target = Path(save_directory, self.get_image_name()) - if not self.force_pull: - if target.exists(): - _LOGGER.info(f"Already cached {self.req} with Singularity.") - return - # Otherwise check other paths old versions may have placed it at. - alternate_targets = [ - Path(save_directory, img) for img in self.get_alternate_image_names() - ] - for alt_target in alternate_targets: - if alt_target.exists(): - _LOGGER.info( - f"Already cached {self.req} with Singularity using a previous caching scheme." - ) - return + return Path(save_directory, self.get_image_name()) + + def _promote(self, source: Path, target: Path) -> None: + """ + Promote an image from an alternate path to a main path. + + Will hardlink source at target if possible, and copy it there + otherwise. + """ + try: + target.hardlink_to(source) + except NotImplementedError: + # Use a temporary file to make sure the replacement is atomic. + # Don't use mkstemp because we might want the file to be readable + # by other users. + temp_target = target.with_suffix(f".tmp.{uuid4()}") + shutil.copy(source, temp_target) + temp_target.replace(target) + + def save_docker_image_from_cache( + self, + target: Path, + search_paths: list[Path | str] | None = None, + ) -> bool: + """ + Put the image we need at our destination path, if we have it locally. + + :param target: The destination path. + + Checks target, plus under save_directory, plus inder the directories in + search_paths if provided. + + :returns: True if the image was found and put in place, and False otherwise. + + If force_pull is set, does nothing and returns False. + """ + if self.force_pull: + # Never use the cache, always pull. + return False + + save_directory: str | Path + if self.save_directory: + save_directory = self.save_directory + if target.exists(): + _LOGGER.info(f"Already cached {self.req} with Singularity.") + return True + # Otherwise check other paths old versions may have placed it at. + + # We want to find any of these names + names = [target.name] + self.get_alternate_image_names() + + # Recursively look under any of these paths + if search_paths is None: + search_paths = [] + search_paths = [save_directory] + search_paths + + # Find the source file to promote to the target path + source: Path | None = None + for search_path in search_paths: + for dirpath, _subdirs, files in os.walk(search_path): + # We need to check our filenames in priority order + file_set = set(files) + for wanted in names: + if wanted in file_set: + path = Path(dirpath) / wanted + if os.path.isfile(path): + _LOGGER.info( + "Using local copy of Singularity image %s found in %s", + wanted, + dirpath, + ) + source = path + break + if source is not None: + break + if source is not None: + break + + if source: + self._promote(source, target) + return True + return False + + def save_docker_image(self) -> None: + """ + Pull down the Docker software container image and save it in the Singularity image format. + + Uses the cache if possible. + """ + + target = self.find_destination_path() + if self.save_docker_image_from_cache(target): + return _LOGGER.info(f"Pulling {self.req} with Singularity...") cmd_pull = [ self.cmd, "pull", ] + env_pull = os.environ.copy() + + if is_singularity_version_2_6(): + env_pull["SINGULARITY_PULLFOLDER"] = str(target.parent) + target_name = target.name + else: + target_name = str(target) + if self.force_pull: cmd_pull.append("--force") cmd_pull.extend( [ "--name", - str(target), + target_name, f"docker://{self.req}", ] ) - ImagePuller._run_command_pull(cmd_pull) + ImagePuller._run_command_pull(cmd_pull, env_pull) _LOGGER.info(f"Image successfully pulled: {target}") From 6caa27f69d090b98c346613f82115f7adcb9ca54 Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Tue, 4 Aug 2026 14:54:33 -0700 Subject: [PATCH 15/16] Make typing more specific to avoid variance complaints --- src/cwl_utils/image_puller.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/cwl_utils/image_puller.py b/src/cwl_utils/image_puller.py index b2f735da..c9705c3f 100644 --- a/src/cwl_utils/image_puller.py +++ b/src/cwl_utils/image_puller.py @@ -223,7 +223,7 @@ def _promote(self, source: Path, target: Path) -> None: def save_docker_image_from_cache( self, target: Path, - search_paths: list[Path | str] | None = None, + search_paths: list[Path] | None = None, ) -> bool: """ Put the image we need at our destination path, if we have it locally. @@ -253,13 +253,13 @@ def save_docker_image_from_cache( names = [target.name] + self.get_alternate_image_names() # Recursively look under any of these paths - if search_paths is None: - search_paths = [] - search_paths = [save_directory] + search_paths + to_search: list[Path | str] = [save_directory] + if search_paths: + to_search.extend(search_paths) # Find the source file to promote to the target path source: Path | None = None - for search_path in search_paths: + for search_path in to_search: for dirpath, _subdirs, files in os.walk(search_path): # We need to check our filenames in priority order file_set = set(files) From 2ec2d8044a76649a72dff4b12a8d8a08d1664e94 Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Tue, 4 Aug 2026 15:05:03 -0700 Subject: [PATCH 16/16] Appease docsting police --- src/cwl_utils/image_puller.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/cwl_utils/image_puller.py b/src/cwl_utils/image_puller.py index c9705c3f..a7c3ee70 100644 --- a/src/cwl_utils/image_puller.py +++ b/src/cwl_utils/image_puller.py @@ -195,9 +195,7 @@ def get_alternate_image_names(self) -> list[str]: return possibilities def find_destination_path(self) -> Path: - """ - Find the path where the image belongs. - """ + """Find the path where the image belongs.""" save_directory: str | Path if self.save_directory: save_directory = self.save_directory @@ -290,7 +288,6 @@ def save_docker_image(self) -> None: Uses the cache if possible. """ - target = self.find_destination_path() if self.save_docker_image_from_cache(target): return