diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..e9ceba7 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,25 @@ +name: Tests + +on: + push: + branches: [main] + pull_request: + +permissions: + contents: read + +jobs: + pytest: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v5 + - uses: actions/setup-python@v6 + with: + python-version: "3.11" + - name: Install test dependencies + run: | + python -m pip install --upgrade pip + python -m pip install pytest pytest-cov + - name: Run full test suite with coverage gate + run: python -m pytest -q diff --git a/README.md b/README.md index 824e1a0..a680f36 100644 --- a/README.md +++ b/README.md @@ -138,6 +138,33 @@ Highlights include: ## Command-line interface +### Inspect before extracting (read-only) + +Preview the *exact selection plan* without creating or overwriting the output +or the optional JSON report: + +```bash +python -m services.cli ./project --dry-run --extensions py md +``` + +Generate a machine-readable preview suitable for CI, shell automation, +or an AI assistant: + +```bash +python -m services.cli ./project --dry-run --preview-format json > selection.json +``` + +The preview lists source-relative paths, byte sizes, totals, and any warning +about an existing destination. Preview and extraction share the same file +selection logic, including deterministic ordering and exclusion rules. +README/specification files no longer bypass filters. File symlinks are +excluded to avoid reading outside the selected tree. + +**Note:** Preview does not inspect file contents, estimate model tokens, +or guarantee that a selected file can be decoded as UTF-8. It never writes +the configured output/report paths; shell redirection in the JSON example +is performed by the shell. + ### Useful examples Extract Python, Markdown, and JSON files: @@ -200,6 +227,8 @@ python -m services.cli ./project \ | `--include-hidden` | flag | off | Traverse hidden files and folders. | | `--exclude-files` | list | empty | File-name patterns to skip. | | `--exclude-folders` | list | empty | Folder-name patterns to skip. | +| `--dry-run` | flag | off | Show proposed extraction without writing output or report. | +| `--preview-format` | choice | `text` | Preview as `text` or `json`. | | `--output` | path | `extraction.txt` | Combined text output. | | `--report` | path | none | Write a JSON extraction report. | | `--max-file-size-mb` | integer | auto | Soft warning threshold; large files are still streamed. | @@ -220,7 +249,9 @@ The processing layer: - Skips unreadable or non-UTF-8 files cleanly instead of corrupting the combined output. - Tracks processed/skipped files, throughput, queue depth, dropped status messages, large-file warnings, and completion timestamps. - Supports cooperative cancellation during traversal and file streaming. -- Preserves terminal state messages under queue pressure. +- Preserves terminal state messages under queue pressure. +- Stages output in a temporary sibling file and atomically replaces the + destination on success; failed or cancelled runs preserve existing output. That instrumentation is also exposed to the CLI logs, making automated jobs easier to observe and diagnose. diff --git a/processor.py b/processor.py index 91d8374..5d3ffe1 100644 --- a/processor.py +++ b/processor.py @@ -5,6 +5,7 @@ import fnmatch import hashlib import os +import tempfile from copy import deepcopy from datetime import datetime from queue import Empty, Full, Queue @@ -523,6 +524,7 @@ def _iter_eligible_file_paths( exclude_files: Sequence[str], exclude_folders: Sequence[str], output_file_abs: str, + temp_file_abs: str | None = None, is_cancelled: Callable[[], bool] | None = None, ) -> Iterable[str]: """Yield eligible file paths while respecting cancellation hooks.""" @@ -571,7 +573,17 @@ def _iter_eligible_file_paths( ) ] - dirs[:] = filtered_dirs + # Stable ordering ensures preview and extraction produce the same plan. + dirs[:] = sorted(filtered_dirs) + filtered_files.sort( + key=lambda name: ( + 0 + if os.path.abspath(root) == os.path.abspath(folder_path) + and name in SPECIFICATION_FILES + else 1, + name, + ) + ) for file_name in filtered_files: if is_cancelled and is_cancelled(): @@ -580,7 +592,10 @@ def _iter_eligible_file_paths( ) file_path = os.path.join(root, file_name) - if os.path.abspath(file_path) == output_file_abs: + # File symlinks can otherwise expose data outside the selected tree. + if os.path.islink(file_path): + continue + if os.path.abspath(file_path) in (output_file_abs, temp_file_abs): continue if file_path in self.processed_files or file_path in seen_paths: continue @@ -640,17 +655,23 @@ def extract_files( if mode_normalised == "inclusion" and not normalised_extensions: normalised_extensions = tuple(COMMON_EXTENSIONS) + temp_output_path: str | None = None try: output_file_abs = os.path.abspath(output_file_name) - with open(output_file_name, "w", encoding="utf-8") as output: - skipped_specs = self.process_specifications( - folder_path, - output, - is_cancelled=is_cancelled, - ) - skipped_count += skipped_specs - + # Stage output next to its destination for atomic replacement. + # Incomplete/failed runs preserve the previous output. + with tempfile.NamedTemporaryFile( + mode="w", + encoding="utf-8", + dir=os.path.dirname(output_file_abs), + prefix=".file-extractor-", + suffix=".tmp", + delete=False, + ) as output: + temp_output_path = output.name + # All files, including README/specifications, must pass the same + # inclusion/exclusion filter. Prioritisation lives in traversal. # Fix: Q-102 - pre-compute eligible files but fall back when memory is scarce. total_files: int iteration_source: Iterable[str] @@ -664,6 +685,7 @@ def extract_files( exclude_files=exclude_files, exclude_folders=exclude_folders, output_file_abs=output_file_abs, + temp_file_abs=temp_output_path, is_cancelled=is_cancelled, ) ) @@ -685,6 +707,7 @@ def extract_files( exclude_files=exclude_files, exclude_folders=exclude_folders, output_file_abs=output_file_abs, + temp_file_abs=temp_output_path, is_cancelled=is_cancelled, ) else: @@ -721,15 +744,18 @@ def extract_files( if progress_callback: progress_callback(processed_count, total_files) - completion_message = ( - f"Extraction complete. Processed {processed_count} files. " - f"Results written to {output_file_name}." + # The complete staged output is now safely closed and committed. + os.replace(temp_output_path, output_file_abs) + temp_output_path = None + completion_message = ( + f"Extraction complete. Processed {processed_count} files. " + f"Results written to {output_file_name}." + ) + if skipped_count: + completion_message += ( + f" Skipped {skipped_count} files due to safeguards." ) - if skipped_count: - completion_message += ( - f" Skipped {skipped_count} files due to safeguards." - ) - self._enqueue_message("info", completion_message) + self._enqueue_message("info", completion_message) except Exception as exc: error_msg = f"Error during extraction: {exc}" @@ -800,6 +826,12 @@ def extract_files( f"skipped={skipped_count}" ), ) + finally: + if temp_output_path is not None: + try: + os.unlink(temp_output_path) + except FileNotFoundError: + pass __all__ = [ diff --git a/services/cli.py b/services/cli.py index aa4e180..75ea2d6 100644 --- a/services/cli.py +++ b/services/cli.py @@ -69,6 +69,8 @@ class CLIOptions: max_file_size_mb: int | None = None poll_interval: float = DEFAULT_POLL_INTERVAL log_level: str = "INFO" + dry_run: bool = False + preview_format: str = "text" # Fix: Q-101 - support nested sequences from repeated --extensions flags. @@ -155,6 +157,17 @@ def build_parser() -> argparse.ArgumentParser: default=(), help="Folder name patterns to exclude", ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Show the files that would be extracted without writing anything", + ) + parser.add_argument( + "--preview-format", + choices=("text", "json"), + default="text", + help="Output format for --dry-run (default: text)", + ) parser.add_argument( "--output", type=Path, @@ -217,6 +230,8 @@ def parse_arguments(argv: Sequence[str] | None = None) -> CLIOptions: max_file_size_mb=args.max_file_size_mb, poll_interval=args.poll_interval, log_level=args.log_level, + dry_run=args.dry_run, + preview_format=args.preview_format, ) @@ -285,6 +300,25 @@ def run_cli( ) -> int: """Execute the extraction using the service layer and return an exit code.""" + if options.dry_run: + from services.preview import build_preview, format_preview + + try: + result = build_preview( + folder_path=options.folder_path, + mode=options.mode, + include_hidden=options.include_hidden, + extensions=options.extensions, + exclude_files=options.exclude_files, + exclude_folders=options.exclude_folders, + output_file=options.output_file, + ) + except (OSError, ValueError) as exc: + print(f"Preview failed: {exc}", file=sys.stderr) + return 1 + print(format_preview(result, options.preview_format)) + return 0 + if configure_logger_handler and not logger.handlers: stream_handler = logging.StreamHandler(stream=sys.stdout) configure_logging( diff --git a/services/preview.py b/services/preview.py new file mode 100644 index 0000000..d9a5221 --- /dev/null +++ b/services/preview.py @@ -0,0 +1,112 @@ +"""Read-only, deterministic preview of a proposed extraction run. + +Both preview and extraction use FileProcessor._iter_eligible_file_paths so that +there are no independent, diverging file-selection implementations. +""" + +from __future__ import annotations + +import json +import os +from pathlib import Path +from queue import Queue +from typing import Any, Sequence + +from constants import COMMON_EXTENSIONS +from processor import FileProcessor + + +def build_preview( + *, + folder_path: Path, + mode: str, + include_hidden: bool, + extensions: Sequence[str], + exclude_files: Sequence[str], + exclude_folders: Sequence[str], + output_file: Path, +) -> dict[str, Any]: + """Produce an exact file-selection plan without creating or modifying files.""" + root = folder_path.resolve(strict=True) + if not root.is_dir(): + raise ValueError(f"Not a directory: {root}") + + normalized_mode = mode.lower() + if normalized_mode not in {"inclusion", "exclusion"}: + raise ValueError(f"Unsupported extraction mode: {mode}") + + selected_extensions = ( + tuple(COMMON_EXTENSIONS) + if normalized_mode == "inclusion" and not extensions + else tuple(extensions) + ) + processor = FileProcessor(Queue()) + entries: list[dict[str, Any]] = [] + warnings: list[str] = [] + output_abs = os.path.abspath(output_file) + + for file_path in processor._iter_eligible_file_paths( + folder_path=str(root), + mode=normalized_mode, + include_hidden=include_hidden, + extensions=selected_extensions, + exclude_files=exclude_files, + exclude_folders=exclude_folders, + output_file_abs=output_abs, + ): + try: + size = os.path.getsize(file_path) + except OSError as exc: + warnings.append(f"Could not stat {file_path}: {exc}") + continue + entries.append( + { + "path": Path(file_path).relative_to(root).as_posix(), + "size_bytes": size, + } + ) + + if output_file.exists(): + warnings.append(f"Extraction will overwrite existing output: {output_file}") + + total_bytes = sum(entry["size_bytes"] for entry in entries) + return { + "source": str(root), + "output": str(output_file), + "mode": normalized_mode, + "file_count": len(entries), + "total_bytes": total_bytes, + "files": entries, + "warnings": warnings, + "note": ( + "Preview reports selection and source sizes; unreadable or non-UTF-8 " + "files may be skipped during extraction." + ), + } + + +def format_preview(preview: dict[str, Any], format_name: str = "text") -> str: + """Render the preview as readable terminal text or machine-readable JSON.""" + if format_name == "json": + return json.dumps(preview, indent=2, ensure_ascii=False) + if format_name != "text": + raise ValueError(f"Unsupported preview format: {format_name}") + + lines = [ + "File Extractor Pro — read-only preview", + f"Source: {preview['source']}", + f"Output: {preview['output']}", + f"Selected: {preview['file_count']} files, " + f"{preview['total_bytes']} source bytes", + "", + ] + for entry in preview["files"]: + lines.append(f" {entry['path']} ({entry['size_bytes']} bytes)") + if preview["warnings"]: + lines.extend(["", "Warnings:"]) + lines.extend(f" - {warning}" for warning in preview["warnings"]) + lines.extend(["", str(preview["note"])]) + return "\n".join(lines) + + +__all__ = ["build_preview", "format_preview"] diff --git a/tests/test_preview.py b/tests/test_preview.py new file mode 100644 index 0000000..01119e7 --- /dev/null +++ b/tests/test_preview.py @@ -0,0 +1,169 @@ +"""Regression tests for zero-write previews and matching extraction selection.""" + +from __future__ import annotations + +import json +import os +from pathlib import Path +from queue import Queue + +import pytest + +from processor import FileProcessor +from services.cli import main +from services.preview import build_preview, format_preview + + +def _preview(root: Path, output: Path, **changes): + args = { + "folder_path": root, + "mode": "inclusion", + "include_hidden": False, + "extensions": (".md", ".py"), + "exclude_files": (), + "exclude_folders": (), + "output_file": output, + } + args.update(changes) + return build_preview(**args) + + +def test_preview_lists_stable_order_and_size_without_writing(tmp_path: Path) -> None: + (tmp_path / "src").mkdir() + (tmp_path / "src" / "z.py").write_text("z", encoding="utf-8") + (tmp_path / "src" / "a.py").write_text("abc", encoding="utf-8") + (tmp_path / "README.md").write_text("readme", encoding="utf-8") + out = tmp_path / "bundle.txt" + first = _preview(tmp_path, out) + second = _preview(tmp_path, out) + + assert first == second + assert [e["path"] for e in first["files"]] == [ + "README.md", "src/a.py", "src/z.py" + ] + assert first["total_bytes"] == 10 + assert not out.exists() + assert json.loads(format_preview(first, "json")) == first + + +def test_preview_matches_extractor_when_readme_is_excluded(tmp_path: Path) -> None: + (tmp_path / "README.md").write_text("SHOULD-NOT-BE-EXPORTED", encoding="utf-8") + (tmp_path / "source.py").write_text("print('ok')", encoding="utf-8") + out = tmp_path / "bundle.txt" + selected = _preview(tmp_path, out, extensions=(".py",), exclude_files=("README.md",)) + assert [e["path"] for e in selected["files"]] == ["source.py"] + + processor = FileProcessor(Queue()) + processor.extract_files( + folder_path=str(tmp_path), + mode="inclusion", + include_hidden=False, + extensions=(".py",), + exclude_files=("README.md",), + exclude_folders=(), + output_file_name=str(out), + ) + extracted = out.read_text(encoding="utf-8") + assert "SHOULD-NOT-BE-EXPORTED" not in extracted + assert "print('ok')" in extracted + + +def test_cli_dry_run_json_is_read_only(tmp_path: Path, capsys) -> None: + (tmp_path / "file.py").write_text("test", encoding="utf-8") + out = tmp_path / "output.txt" + report = tmp_path / "report.json" + code = main([ + str(tmp_path), + "--extensions", "py", + "--dry-run", + "--preview-format", "json", + "--output", str(out), + "--report", str(report), + ]) + assert code == 0 + payload = json.loads(capsys.readouterr().out) + assert payload["file_count"] == 1 + assert payload["files"][0]["path"] == "file.py" + assert not out.exists() + assert not report.exists() + + +def test_preview_never_mutates_existing_output(tmp_path: Path) -> None: + (tmp_path / "source.py").write_text("secret", encoding="utf-8") + out = tmp_path / "bundle.txt" + out.write_text("keep-me", encoding="utf-8") + preview = _preview(tmp_path, out) + assert preview["file_count"] == 1 + assert preview["warnings"] + assert out.read_text(encoding="utf-8") == "keep-me" + + +def test_file_symlink_outside_tree_is_excluded(tmp_path: Path) -> None: + outside = tmp_path.parent / (tmp_path.name + "-outside.py") + outside.write_text("outside", encoding="utf-8") + link = tmp_path / "link.py" + try: + link.symlink_to(outside) + except (OSError, NotImplementedError): + pytest.skip("Symlinks are unavailable") + try: + result = _preview(tmp_path, tmp_path / "out.txt", extensions=(".py",)) + assert result["file_count"] == 0 + finally: + link.unlink() + outside.unlink() + + +def test_preview_invalid_folder_returns_error(tmp_path: Path, capsys) -> None: + code = main([str(tmp_path / "missing"), "--dry-run"]) + assert code == 1 + assert "Preview failed:" in capsys.readouterr().err + + +def test_preview_rejects_invalid_format(tmp_path: Path) -> None: + with pytest.raises(ValueError): + format_preview(_preview(tmp_path, tmp_path / "out.txt"), "xml") + + +def test_failed_extraction_preserves_previous_output(tmp_path: Path, monkeypatch) -> None: + """An unexpected failure must not destroy a pre-existing result.""" + (tmp_path / "one.py").write_text("original", encoding="utf-8") + out = tmp_path / "bundle.txt" + out.write_text("previous successful output", encoding="utf-8") + worker = FileProcessor(Queue()) + + def fail_file(*args, **kwargs): + raise RuntimeError("simulated failure after opening temporary output") + + monkeypatch.setattr(worker, "process_file", fail_file) + with pytest.raises(RuntimeError, match="simulated failure"): + worker.extract_files( + folder_path=str(tmp_path), + mode="inclusion", + include_hidden=True, + extensions=(".py",), + exclude_files=(), + exclude_folders=(), + output_file_name=str(out), + ) + assert out.read_text(encoding="utf-8") == "previous successful output" + assert not list(tmp_path.glob(".file-extractor-*.tmp")) + + +def test_atomic_temporary_file_is_never_included(tmp_path: Path) -> None: + (tmp_path / "a.py").write_text("small sample", encoding="utf-8") + out = tmp_path / "bundle.txt" + worker = FileProcessor(Queue()) + worker.extract_files( + folder_path=str(tmp_path), + mode="inclusion", + include_hidden=True, + extensions=("*",), + exclude_files=(), + exclude_folders=(), + output_file_name=str(out), + ) + result = out.read_text(encoding="utf-8") + assert "a.py" in result + assert ".file-extractor-" not in result + assert not list(tmp_path.glob(".file-extractor-*.tmp"))