Add soulcap-lookup, soulcap-tokens, type checking, and CI improvements

.claude/hooks/validate_report_quotes.py

@@ -86,7 +86,9 @@ def main() -> int:
     try:
         from soulcap_cl_mapping import report_validator
     except Exception as exc:  # pragma: no cover - defensive
-        print(f"validate_report_quotes: import failed ({exc}); allowing", file=sys.stderr)
+        print(
+            f"validate_report_quotes: import failed ({exc}); allowing", file=sys.stderr
+        )
         return 0
 
     run_id = path.parent.name
@@ -104,7 +106,10 @@ def main() -> int:
 
         evidence = snippet_cache.load_snippet_texts(run_id, qid, None)
     except Exception as exc:  # pragma: no cover - defensive
-        print(f"validate_report_quotes: cache read failed ({exc}); allowing", file=sys.stderr)
+        print(
+            f"validate_report_quotes: cache read failed ({exc}); allowing",
+            file=sys.stderr,
+        )
         return 0
 
     if not evidence:

.github/dependabot.yml

@@ -0,0 +1,11 @@
+version: 2
+updates:
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+
+  - package-ecosystem: "pip"
+    directory: "/"
+    schedule:
+      interval: "weekly"

.github/workflows/tests.yml

@@ -6,6 +6,31 @@ on:
     branches: [main]
 
 jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          enable-cache: true
+
+      - name: Set up Python
+        run: uv python install 3.12
+
+      - name: Install dependencies
+        run: uv sync --frozen --extra dev
+
+      - name: Lint
+        run: uv run ruff check .
+
+      - name: Format check
+        run: uv run ruff format --check .
+
+      - name: Type check
+        run: uv run mypy src/soulcap_cl_mapping
+
   test:
     runs-on: ubuntu-latest
     strategy:
@@ -25,10 +50,7 @@ jobs:
         run: uv python install ${{ matrix.python-version }}
 
       - name: Install dependencies
-        run: uv sync --extra dev
-
-      - name: Lint
-        run: uv run ruff check .
+        run: uv sync --frozen --extra dev
 
       - name: Run tests (with coverage gate)
         run: uv run pytest

.pre-commit-config.yaml

@@ -0,0 +1,7 @@
+repos:
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.15.19
+    hooks:
+      - id: ruff
+        args: [--fix]
+      - id: ruff-format

pyproject.toml

@@ -19,12 +19,16 @@ soulcap-validate = "soulcap_cl_mapping.marker_syntax:main"
 soulcap-cache = "soulcap_cl_mapping.snippet_cache:main"
 soulcap-validate-report = "soulcap_cl_mapping.report_validator:main"
 soulcap-cl-pro = "soulcap_cl_mapping.cl_pro:main"
+soulcap-lookup = "soulcap_cl_mapping.ols4_lookup:main"
+soulcap-tokens = "soulcap_cl_mapping.marker_syntax:tokens_main"
 
 [project.optional-dependencies]
 dev = [
     "pytest>=8.0",
     "pytest-cov>=5.0",
     "ruff>=0.6",
+    "mypy>=1.10",
+    "pandas-stubs>=2.2",
 ]
 
 [build-system]
@@ -41,3 +45,9 @@ addopts = "--cov=soulcap_cl_mapping --cov-report=term-missing --cov-fail-under=8
 [tool.coverage.run]
 branch = true
 source = ["soulcap_cl_mapping"]
+
+[tool.mypy]
+python_version = "3.12"
+disallow_untyped_defs = true
+warn_return_any = true
+warn_unused_ignores = true

reports/marker_validation.md

@@ -1,39 +1,41 @@
-# Marker Syntax Validation
+# Marker Syntax Validation
 
-> **Auto-generated** on each `soulcap-sync` by the EBNF validator (`soulcap_cl_mapping.marker_syntax`). Do not edit by hand — edit the Google Sheet master and re-sync.
+> **Auto-generated** on each `soulcap-sync` by the EBNF validator (`soulcap_cl_mapping.marker_syntax`). Do not edit by hand — edit the Google Sheet master and re-sync.
 
 - Source: `marker_combinations.csv`
-- Cells checked: **256**
-- Invalid cells: **24**
-- Grammar: [../MARKER_SYNTAX.md](../MARKER_SYNTAX.md) §2
+- Cells checked: **390**
+- Invalid cells: **26**
+- Grammar: [../MARKER_SYNTAX.md](../MARKER_SYNTAX.md) §2
 
 ## Invalid cells
 
 | Sheet row | Subset | Column | Error | Value |
 |-----------|--------|--------|-------|-------|
-| 8 | ILC | Ideal phenotypic markers | invalid marker token "don't" | `Confirm that ILC don't bind to CD14/CD19/CD56/CD11c/CD123` |
-| 16 | Basophil | Required phenotypic markers | top-level '\|' must be inside a group | `live/ CD45+ CD33+\|CD123+) (CD193+\|FceR1a+\|HLA-DR-\|CD303-)` |
-| 18 | Mono | Ideal phenotypic markers | top-level '\|' must be inside a group | `CD64+/-\|CD33+/-)` |
-| 20 | NCMo | Ideal phenotypic markers | top-level '\|' must be inside a group | `CD64+/-\|CD33+/-)` |
-| 32 | Naive | Required phenotypic markers | missing space before 'CD27-' | `live/ CD45+ (CD20+\|CD19+)CD27- (CD38lo/-\|CD10-) CD24- IgD+` |
-| 46 | iNKT | Required phenotypic markers | mixed '\|' and space in a group | `CD45+ CD3+ (TCRab+\|TCRgd-) (TCRVa24-Ja18+\|TCRva24+\|CD1d-a-GalCer Tetramer+) TCR VB11+` |
-| 47 | CD4 Con T | Required phenotypic markers | mixed '\|' and space in a group | `CD45+ CD3+ (TCRab+\|TCRgd-) [(TCRVa24-Ja18+\|TCRva24+\|CD1d-a-GalCer Tetramer+) TCRVB11+]- [(TCRVa7.2+\|MR1 Tetramer+) CD161+]- CD4+` |
-| 48 | CD8 ConT | Required phenotypic markers | mixed '\|' and space in a group | `CD45+ CD3+ (TCRab+\|TCRgd-) [(TCRVa24-Ja18+\|TCRva24+\|CD1d-a-GalCer Tetramer+) TCRVB11+]- [(TCRVa7.2+\|MR1 Tetramer+) CD161+]- CD8+` |
-| 49 | MAIT | Required phenotypic markers | mixed '\|' and space in a group | `CD45+ CD3+ (TCRab+\|TCRgd-) (TCRVa7.2+\|MR1 Tetramer+) CD161+` |
-| 50 | CD4/CD8 ConT memory | Required phenotypic markers | mixed '\|' and space in a group | `CD45+ CD3+ (TCRab+\|TCRgd-)  (TCRVa24-Ja18\|TCRva24\|CD1d-a-GalCer Tetramer) TCRVB11 (TCRVa7.2\|MR1 Tetramer) CD161 CD4 CD8 CD45RA CD197` |
-| 51 | CD4/CD8 ConT helper | Required phenotypic markers | mixed '\|' and space in a group | `CD45+ CD3+ (TCRab+\|TCRgd-)  (TCRVa24-Ja18\|TCRva24\|CD1d-a-GalCer Tetramer) TCRVB11 (TCRVa7.2\|MR1 Tetramer) CD161 CD4 CD8 (CD183\|CXCR3) (CD185\|CXCR5) (CD194\|CCR4) (CD196\|CCR6)` |
-| 52 | ConT Treg-like | Required phenotypic markers | mixed '\|' and space in a group | `CD45+ CD3+ (TCRab+\|TCRgd-) [(TCRVa24-Ja18+\|TCRva24+\|CD1d-a-GalCer Tetramer+) TCRVB11+]- [(TCRVa7.2+\|MR1 Tetramer+) CD161+]- CD4+ CD25+ CD127lo/-` |
-| 53 | ConT Tfh-like | Required phenotypic markers | mixed '\|' and space in a group | `CD45+ CD3+ (TCRab+\|TCRgd-) [(TCRVa24-Ja18+\|TCRva24+\|CD1d-a-GalCer Tetramer+)+ TCRVB11+]- [(TCRVa7.2+\|MR1 Tetramer+) CD161+]- CD4+ [CD25+ CD127lo/-]- CD185+` |
-| 54 | Th1-like | Required phenotypic markers | mixed '\|' and space in a group | `CD45+ CD3+ (TCRab+\|TCRgd-) [(TCRVa24-Ja18+\|TCRva24+\|CD1d-a-GalCer Tetramer+)+ TCRVB11+]- [(TCRVa7.2+\|MR1 Tetramer+) CD161+]- CD4+ [CD25+ CD127lo/-]- CD185- CD183+ CD196- CD194-` |
-| 55 | Th2-like | Required phenotypic markers | mixed '\|' and space in a group | `CD45+ CD3+ (TCRab+\|TCRgd-) [(TCRVa24-Ja18+\|TCRva24+\|CD1d-a-GalCer Tetramer+)+ TCRVB11+]- [(TCRVa7.2+\|MR1 Tetramer+) CD161+]- CD4+ [CD25+ CD127lo/-]- CD185- CD183- CD196- CD194+ CD294+` |
-| 56 | Th9-like | Required phenotypic markers | mixed '\|' and space in a group | `CD45+ CD3+ (TCRab+\|TCRgd-) [(TCRVa24-Ja18+\|TCRva24+\|CD1d-a-GalCer Tetramer+)+ TCRVB11+]- [(TCRVa7.2+\|MR1 Tetramer+) CD161+]- CD4+ [CD25+ CD127lo/-]- CD185- CD183- CD196- CD194+` |
-| 57 | Th17-like | Required phenotypic markers | mixed '\|' and space in a group | `CD45+ CD3+ (TCRab+\|TCRgd-) [(TCRVa24-Ja18+\|TCRva24+\|CD1d-a-GalCer Tetramer+)+ TCRVB11+]- [(TCRVa7.2+\|MR1 Tetramer+) CD161+]- CD4+ [CD25+ CD127lo/-]- CD185- CD183- CD196+ CD194+/-` |
-| 72 | Vd2 | Required phenotypic markers | invalid marker token '2+' | `CD45+ CD3+ (TCRab-\|TCRgd+) TCR V delta 2+ TCR V gamma 9-` |
-| 73 | Vd3 | Required phenotypic markers | invalid marker token '1-' | `CD45+ CD3+ (TCRab-\|TCRgd+) TCR V delta 1- TCR V delta 2-` |
-| 73 | Vd3 | Ideal phenotypic markers | invalid marker token '9+/-' | `TCR V gamma 9+/-` |
-| 74 | Vd1 | Required phenotypic markers | invalid marker token '1+' | `CD45+ CD3+ (TCRab-\|TCRgd+) TCR V delta 1+` |
-| 74 | Vd1 | Ideal phenotypic markers | invalid marker token '2-' | `TCR V delta 2- TCR V gamma 9+/-` |
-| 75 | Vg9 | Required phenotypic markers | invalid marker token '9+' | `CD45+ CD3+ (TCRab-\|TCRgd+) TCR V gamma 9+` |
-| 75 | Vg9 | Ideal phenotypic markers | invalid marker token '1-' | `TCR V delta 1- TCR V delta 2-` |
+| 18 |  | Required exclusion | unexpected ')' in group | `CD56- CD19- ([CD123hi]- CD303-) ([CD123hi\|CD193-)` |
+| 19 |  | Required exclusion | unexpected ')' in group | `CD56- CD19- ([CD123hi]- CD303-) ([CD123hi\|CD193-)` |
+| 20 |  | Required exclusion | unexpected ')' in group | `CD56- CD19- ([CD123hi]- CD303-) ([CD123hi\|CD193-)` |
+| 21 |  | Required exclusion | unexpected ')' in group | `CD56- CD19- ([CD123hi]- CD303-) ([CD123hi\|CD193-)` |
+| 60 |  | Required phenotypic markers | mixed '\|' and space in a group | `CD45+ CD3+ (TCRab+\|TCRgd-) (TCRVa24-Ja18+\|TCRVa24+\|CD1d-a-GalCer Tetramer+) TCR Vb11+` |
+| 75 |  | Required phenotypic markers | mixed '\|' and space in a group | `CD45+ CD3+ (TCRab+\|TCRgd-) (TCRVa7.2+\|MR1 Tetramer+) CD161+` |
+| 76 |  | Required phenotypic markers | mixed '\|' and space in a group | `CD45+ CD3+ (TCRab+\|TCRgd-) [(TCRVa7.2+\|MR1 Tetramer+) CD161+]- [(TCRVa24-Ja18+\|TCRVa24+\|CD1d-a-GalCer Tetramer+) TCR Vb11+]-` |
+| 77 |  | Required phenotypic markers | mixed '\|' and space in a group | `CD45+ CD3+ (TCRab+\|TCRgd-) [(TCRVa7.2+\|MR1 Tetramer+) CD161+]- [(TCRVa24-Ja18+\|TCRva24+\|CD1d-a-GalCer Tetramer+) TCR VB11+]- CD4+ CD8-` |
+| 78 |  | Required phenotypic markers | mixed '\|' and space in a group | `CD45+ CD3+ (TCRab+\|TCRgd-) [(TCRVa7.2+\|MR1 Tetramer+) CD161+]- [(TCRVa24-Ja18+\|TCRVa24+\|CD1d-a-GalCer Tetramer+) TCR Vb11+]- CD4- CD8+` |
+| 79 |  | Required phenotypic markers | mixed '\|' and space in a group | `CD45+ CD3+ (TCRab+\|TCRgd-) [(TCRVa7.2+\|MR1 Tetramer+) CD161+]- [(TCRVa24-Ja18+\|TCRva24+\|CD1d-a-GalCer Tetramer+) TCR VB11+]- CD4+ CD8+` |
+| 80 |  | Required phenotypic markers | mixed '\|' and space in a group | `CD45+ CD3+ (TCRab+\|TCRgd-) [(TCRVa7.2+\|MR1 Tetramer+) CD161+]- [(TCRVa24-Ja18+\|TCRVa24+\|CD1d-a-GalCer Tetramer+) TCR Vb11+]- CD4- CD8-` |
+| 108 |  | Required phenotypic markers | mixed '\|' and space in a group | `CD45+ CD3+ (TCRab+\|TCRgd-)  (TCRVa24-Ja18\|TCRVa24\|CD1d-a-GalCer Tetramer) TCRVb11 (TCRVa7.2\|MR1 Tetramer) CD161 CD4 CD8 (CD183\|CXCR3) (CD185\|CXCR5) (CD194\|CCR4) (CD196\|CCR6)` |
+| 110 |  | Required phenotypic markers | mixed '\|' and space in a group | `CD45+ CD3+ (TCRab+\|TCRgd-) [(TCRVa24-Ja18+\|TCRVa24+\|CD1d-a-GalCer Tetramer+) TCRVb11+]- [(TCRVa7.2+\|MR1 Tetramer+) CD161+]- CD4+ CD25+ CD127lo/-` |
+| 111 |  | Required phenotypic markers | mixed '\|' and space in a group | `CD45+ CD3+ (TCRab+\|TCRgd-) [(TCRVa24-Ja18+\|TCRVa24+\|CD1d-a-GalCer Tetramer+) TCRVb11+]- [(TCRVa7.2+\|MR1 Tetramer+) CD161+]-  CD4+ [CD25+ CD127lo/-]- CD185+` |
+| 112 |  | Required phenotypic markers | mixed '\|' and space in a group | `CD45+ CD3+ (TCRab+\|TCRgd-) [(TCRVa24-Ja18+\|TCRVa24+\|CD1d-a-GalCer Tetramer+) TCRVb11+]- [(TCRVa7.2+\|MR1 Tetramer+) CD161+]-  CD4+ [CD25+ CD127lo/-]- CD185- CD183+ CD196- CD194-` |
+| 113 |  | Required phenotypic markers | mixed '\|' and space in a group | `CD45+ CD3+ (TCRab+\|TCRgd-) [(TCRVa24-Ja18+\|TCRVa24+\|CD1d-a-GalCer Tetramer+) TCRVb11+]- [(TCRVa7.2+\|MR1 Tetramer+) CD161+]-  CD4+ [CD25+ CD127lo/-]- CD185- CD183- CD196- CD194+ CD294+` |
+| 114 |  | Required phenotypic markers | mixed '\|' and space in a group | `CD45+ CD3+ (TCRab+\|TCRgd-) [(TCRVa24-Ja18+\|TCRVa24+\|CD1d-a-GalCer Tetramer+) TCRVb11+]- [(TCRVa7.2+\|MR1 Tetramer+) CD161+]-  CD4+ [CD25+ CD127lo/-]- CD185- CD183- CD196- CD194+` |
+| 115 |  | Required phenotypic markers | mixed '\|' and space in a group | `CD45+ CD3+ (TCRab+\|TCRgd-) [(TCRVa24-Ja18+\|TCRVa24+\|CD1d-a-GalCer Tetramer+) TCRVb11+]- [(TCRVa7.2+\|MR1 Tetramer+) CD161+]- CD4+ [CD25+ CD127lo/-]- CD185- CD183- CD196+ CD194+/-` |
+| 124 |  | Required phenotypic markers | invalid marker token '2+' | `CD45+ CD3+ (TCRab-\|TCRgd+) TCR V delta 2+ TCR V gamma 9- TCR Vd1-` |
+| 125 |  | Required phenotypic markers | invalid marker token '1-' | `CD45+ CD3+ (TCRab-\|TCRgd+) TCR V delta 1- TCR V delta 2- TCR V gamma 9-` |
+| 125 |  | Ideal phenotypic markers | invalid marker token '9+/-' | `TCR V gamma 9+/-` |
+| 126 |  | Required phenotypic markers | invalid marker token '1+' | `CD45+ CD3+ (TCRab-\|TCRgd+) TCR V delta 1+ TCR V delta 2- TCR V gamma 9+` |
+| 127 |  | Required phenotypic markers | invalid marker token '1+' | `CD45+ CD3+ (TCRab-\|TCRgd+) TCR V delta 1+ TCR V delta 2- TCR V gamma 9+/-` |
+| 127 |  | Ideal phenotypic markers | invalid marker token '2-' | `TCR V delta 2- TCR V gamma 9+/-` |
+| 128 |  | Required phenotypic markers | invalid marker token '1-' | `CD45+ CD3+ (TCRab-\|TCRgd+) TCR V delta 1- TCR V delta 2- TCR V gamma 9+` |
+| 128 |  | Ideal phenotypic markers | invalid marker token '1-' | `TCR V delta 1- TCR V delta 2-` |
 
 See [marker_string_issues.md](marker_string_issues.md) for the broader (non-syntactic) data-quality review.

src/soulcap_cl_mapping/__init__.py

@@ -1,3 +1,8 @@
 """SOULCAP ↔ Cell Ontology mapping utilities."""
 
-__version__ = "0.1.0"
+from importlib.metadata import PackageNotFoundError, version
+
+try:
+    __version__ = version("soulcap-cl-mapping")
+except PackageNotFoundError:
+    __version__ = "unknown"

src/soulcap_cl_mapping/cl_pro.py

@@ -34,7 +34,7 @@
 from collections import defaultdict
 from datetime import date
 from pathlib import Path
-from typing import Callable
+from typing import Any, Callable
 
 import certifi
 from SPARQLWrapper import JSON, SPARQLWrapper
@@ -108,7 +108,7 @@ def run_sparql(
     wrapper.setTimeout(timeout)
     wrapper.setMethod("POST")
     wrapper.setQuery(query)
-    result = wrapper.query().convert()
+    result: dict[str, Any] = wrapper.query().convert()  # type: ignore[assignment]
     rows: list[dict[str, str]] = []
     for binding in result["results"]["bindings"]:
         rows.append({var: cell["value"] for var, cell in binding.items()})
@@ -392,7 +392,7 @@ def fetch_pr_metadata(query_fn: QueryFn = run_sparql) -> dict[str, dict]:
     meta: dict[str, dict] = {}
     for row in query_fn(PR_METADATA_QUERY):
         xrefs = _split(row.get("xrefs", ""))
-        entry = {
+        entry: dict[str, Any] = {
             "label": (row.get("label", "") or "").split("|")[0],
             "taxa": [TAXA[t] for t in _split(row.get("taxon", "")) if t in TAXA],
             "exact": _split(row.get("exact_syn", "")),

src/soulcap_cl_mapping/marker_syntax.py

@@ -147,6 +147,7 @@ def _parse_term(self) -> None:
             self._parse_group()
         elif kind == "WORD":
             self._advance()
+            assert text is not None
             _split_word(text)  # validates marker + qualifier
         else:
             raise MarkerSyntaxError(f"expected marker or group, found {text!r}")
@@ -162,7 +163,7 @@ def _parse_group(self) -> None:
         # optional group-level qualifier: a following WORD that is wholly a
         # qualifier, e.g. [HLA-DR+ CD11chi]-
         kind, text = self._peek()
-        if kind == "WORD" and _is_qualifier(text):
+        if kind == "WORD" and text is not None and _is_qualifier(text):
             self._advance()
 
     def _parse_inner(self, close: str) -> None:
@@ -217,7 +218,7 @@ def _scan(path: str | Path) -> tuple[list[dict], int]:
     cols = [c for c in MARKER_COLUMNS if c in df.columns]
     failures: list[dict] = []
     n_cells = 0
-    for idx, row in df.iterrows():
+    for row_num, (_, row) in enumerate(df.iterrows()):
         for col in cols:
             value = row[col]
             if pd.isna(value):
@@ -227,7 +228,7 @@ def _scan(path: str | Path) -> tuple[list[dict], int]:
             if error is not None:
                 failures.append(
                     {
-                        "row": idx + 2,  # +1 header, +1 for 1-based
+                        "row": row_num + 2,  # +1 header, +1 for 1-based
                         "subset": str(row.get("Subset name", "")).strip(),
                         "column": col,
                         "value": str(value).strip(),
@@ -295,7 +296,9 @@ def write_report(
     """Write/overwrite the Markdown validation report; return its path."""
     report_path = Path(report_path)
     report_path.parent.mkdir(parents=True, exist_ok=True)
-    report_path.write_text(render_report(failures, source_name, n_cells))
+    report_path.write_text(
+        render_report(failures, source_name, n_cells), encoding="utf-8"
+    )
     return report_path
 
 
@@ -324,6 +327,108 @@ def validate_marker_csv(
     return failures
 
 
+# --------------------------------------------------------------------------- #
+# Token extraction
+# --------------------------------------------------------------------------- #
+def extract_markers(expr: str) -> set[str]:
+    """Return the distinct marker names in a marker expression.
+
+    Strips qualifiers (e.g. ``CD4+`` → ``CD4``) and grouping symbols.
+    Skips the live-gate prefix and group-level qualifiers. Returns an empty
+    set for empty or invalid expressions — never raises.
+    """
+    try:
+        tokens = _tokenize(expr)
+    except Exception:  # noqa: BLE001
+        return set()
+    markers: set[str] = set()
+    for kind, text in tokens:
+        if kind != "WORD":
+            continue
+        if text == GATE or _is_qualifier(text):
+            continue
+        try:
+            marker, _ = _split_word(text)
+            markers.add(marker)
+        except MarkerSyntaxError:
+            continue
+    return markers
+
+
+def extract_markers_from_csv(path: str | Path) -> list[dict]:
+    """Extract all distinct marker tokens from a Marker Combinations CSV.
+
+    Returns one record per distinct marker token, sorted alphabetically:
+    ``{"marker_token", "source_columns", "cell_types"}`` where the latter
+    two are ``|``-separated strings listing which marker columns and which
+    subset names contain that token.
+    """
+    df = pd.read_csv(path)
+    cols = [c for c in MARKER_COLUMNS if c in df.columns]
+    has_subset = "Subset name" in df.columns
+
+    token_info: dict[str, dict[str, set]] = {}
+    for _, row in df.iterrows():
+        subset = str(row["Subset name"]).strip() if has_subset else ""
+        for col in cols:
+            value = row[col]
+            if pd.isna(value):
+                continue
+            for marker in extract_markers(str(value)):
+                if marker not in token_info:
+                    token_info[marker] = {"columns": set(), "cell_types": set()}
+                token_info[marker]["columns"].add(col)
+                if subset:
+                    token_info[marker]["cell_types"].add(subset)
+
+    return [
+        {
+            "marker_token": token,
+            "source_columns": "|".join(sorted(info["columns"])),
+            "cell_types": "|".join(sorted(info["cell_types"])),
+        }
+        for token, info in sorted(token_info.items())
+    ]
+
+
+def tokens_main(argv: list[str] | None = None) -> int:
+    """CLI: extract distinct marker tokens from the Marker Combinations CSV."""
+    import argparse
+
+    repo_root = Path(__file__).resolve().parents[2]
+    default_csv = repo_root / "data" / "marker_combinations.csv"
+    default_out = repo_root / "marker_mappings" / "marker_tokens.csv"
+
+    parser = argparse.ArgumentParser(
+        prog="soulcap-tokens",
+        description="Extract distinct marker tokens from the Marker Combinations CSV.",
+    )
+    parser.add_argument(
+        "csv",
+        nargs="?",
+        type=Path,
+        default=default_csv,
+        help="Path to marker_combinations.csv (default: data/).",
+    )
+    parser.add_argument(
+        "--out",
+        type=Path,
+        default=default_out,
+        help="Output CSV (default: marker_mappings/marker_tokens.csv).",
+    )
+    args = parser.parse_args(argv)
+
+    if not args.csv.exists():
+        print(f"error: {args.csv} not found — run `soulcap-sync` first.")
+        return 2
+
+    records = extract_markers_from_csv(args.csv)
+    args.out.parent.mkdir(parents=True, exist_ok=True)
+    pd.DataFrame(records).to_csv(args.out, index=False)
+    print(f"Extracted {len(records)} distinct marker token(s) -> {args.out}")
+    return 0
+
+
 def main(argv: list[str] | None = None) -> int:
     import argparse