CI-1901 - Make files and samplesheet optional in preprocess

cirro/helpers/preprocess_dataset.py

@@ -49,6 +49,16 @@ def read_csv(path: str, required_columns=None) -> 'DataFrame':
     return df
 
 
+def _read_optional_csv(path: str, required_columns: list[str]) -> 'DataFrame':
+    """Read a CSV, returning an empty DataFrame (with the required columns) if the file doesn't exist."""
+    try:
+        return read_csv(path, required_columns=required_columns)
+    except FileNotFoundError:
+        import pandas as pd
+        logger.warning(f"File not found: {path}, using an empty table")
+        return pd.DataFrame(columns=required_columns)
+
+
 def read_json(path: str):
     """Read a JSON object from a local file or S3 path."""
     path = _fix_s3_path(path)
@@ -83,13 +93,17 @@ class PreprocessDataset:
     A pandas DataFrame containing all of the metadata assigned to the samples present
     in the input datasets (at the time of analysis).
 
+    Deprecated: this will be removed in a future release.
+
     More info: https://docs.cirro.bio/pipelines/preprocess-script/#dssamplesheet
     """
     files: 'DataFrame'
     """
     A DataFrame containing information on the files that are associated with
     a sample within the input datasets.
 
+    Deprecated: this will be removed in a future release.
+
     More info: https://docs.cirro.bio/pipelines/preprocess-script/#dsfiles
     """
     params: dict
@@ -151,12 +165,12 @@ def from_path(cls, dataset_root: str, config_directory='config'):
         """
         config_directory = Path(dataset_root, config_directory)
 
-        files = read_csv(
+        files = _read_optional_csv(
             str(Path(config_directory, "files.csv")),
             required_columns=["sample", "file"]
         )
 
-        samplesheet = read_csv(
+        samplesheet = _read_optional_csv(
             str(Path(config_directory, "samplesheet.csv")),
             required_columns=["sample"]
         )
@@ -265,6 +279,8 @@ def pivot_samplesheet(
         and the `samplesheet` has columns `sample`, `status`, and `group`, the output
         will have columns `sample`, `fastq_1`, `fastq_2`, `status`, and `group`.
 
+        Deprecated: this will be removed in a future release.
+
         Args:
             index: List[str], used to make the frames new index, defaults to
              `["sampleIndex", "sample", "lane"]`
@@ -319,6 +335,8 @@ def pivot_files(
         paired-end sequencing data where you want to have the columns
         `sample`, `fastq_1`, and `fastq_2` as the output.
 
+        Deprecated: this will be removed in a future release.
+
         Args:
             index: List[str], used to make the frames new index, defaults to
             pivot_columns: str or List[str], columns to pivot on and create the new column,
@@ -393,9 +411,9 @@ def wide_samplesheet(
         """
         Format the samplesheet into a wide format with each sample on a row
 
-        This is a legacy method, please use `pivot_samplesheet` instead.
+        Deprecated: this will be removed in a future release.
         """
-        warnings.warn("`wide_samplesheet` is deprecated, use `pivot_samplesheet` instead.",
+        warnings.warn("`wide_samplesheet` is deprecated and will be removed in a future release.",
                       DeprecationWarning, stacklevel=2)
         if values != "file":
             raise ValueError("The only supported value for `values` is 'file'")

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "cirro"
-version = "1.16.0"
+version = "1.16.1"
 description = "CLI tool and SDK for interacting with the Cirro platform"
 authors = ["Cirro Bio <support@cirro.bio>"]
 license = "MIT"

tests/test_preprocess.py

@@ -1,4 +1,5 @@
 import os
+import tempfile
 import unittest
 from pathlib import Path
 from typing import NamedTuple, Optional
@@ -97,6 +98,19 @@ def test_load_running(self):
         df = ds.pivot_samplesheet()
         self.assertGreater(df.shape[0], 0)
 
+    def test_load_missing_samplesheet_and_files(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            config_dir = Path(tmp, 'config')
+            config_dir.mkdir()
+            Path(config_dir, 'params.json').write_text('{"a": 1}')
+            Path(config_dir, 'metadata.json').write_text('{}')
+
+            ds = PreprocessDataset.from_path(dataset_root=tmp)
+
+        self.assertEqual(ds.params, {"a": 1})
+        self.assertEqual(ds.samplesheet.shape[0], 0)
+        self.assertEqual(ds.files.shape[0], 0)
+
 
 if __name__ == '__main__':
     unittest.main()