Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 25 additions & 0 deletions .github/workflows/tests.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
name: Tests

on:
push:
branches: [main]
pull_request:

permissions:
contents: read

jobs:
pytest:
runs-on: ubuntu-latest
timeout-minutes: 15
steps:
- uses: actions/checkout@v5
- uses: actions/setup-python@v6
with:
python-version: "3.11"
- name: Install test dependencies
run: |
python -m pip install --upgrade pip
python -m pip install pytest pytest-cov
- name: Run full test suite with coverage gate
run: python -m pytest -q
33 changes: 32 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -138,6 +138,33 @@ Highlights include:

## Command-line interface

### Inspect before extracting (read-only)

Preview the *exact selection plan* without creating or overwriting the output
or the optional JSON report:

```bash
python -m services.cli ./project --dry-run --extensions py md
```

Generate a machine-readable preview suitable for CI, shell automation,
or an AI assistant:

```bash
python -m services.cli ./project --dry-run --preview-format json > selection.json
```

The preview lists source-relative paths, byte sizes, totals, and any warning
about an existing destination. Preview and extraction share the same file
selection logic, including deterministic ordering and exclusion rules.
README/specification files no longer bypass filters. File symlinks are
excluded to avoid reading outside the selected tree.

**Note:** Preview does not inspect file contents, estimate model tokens,
or guarantee that a selected file can be decoded as UTF-8. It never writes
the configured output/report paths; shell redirection in the JSON example
is performed by the shell.

### Useful examples

Extract Python, Markdown, and JSON files:
Expand Down Expand Up @@ -200,6 +227,8 @@ python -m services.cli ./project \
| `--include-hidden` | flag | off | Traverse hidden files and folders. |
| `--exclude-files` | list | empty | File-name patterns to skip. |
| `--exclude-folders` | list | empty | Folder-name patterns to skip. |
| `--dry-run` | flag | off | Show proposed extraction without writing output or report. |
| `--preview-format` | choice | `text` | Preview as `text` or `json`. |
| `--output` | path | `extraction.txt` | Combined text output. |
| `--report` | path | none | Write a JSON extraction report. |
| `--max-file-size-mb` | integer | auto | Soft warning threshold; large files are still streamed. |
Expand All @@ -220,7 +249,9 @@ The processing layer:
- Skips unreadable or non-UTF-8 files cleanly instead of corrupting the combined output.
- Tracks processed/skipped files, throughput, queue depth, dropped status messages, large-file warnings, and completion timestamps.
- Supports cooperative cancellation during traversal and file streaming.
- Preserves terminal state messages under queue pressure.
- Preserves terminal state messages under queue pressure.
- Stages output in a temporary sibling file and atomically replaces the
destination on success; failed or cancelled runs preserve existing output.

That instrumentation is also exposed to the CLI logs, making automated jobs easier to observe and diagnose.

Expand Down
68 changes: 50 additions & 18 deletions processor.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import fnmatch
import hashlib
import os
import tempfile
from copy import deepcopy
from datetime import datetime
from queue import Empty, Full, Queue
Expand Down Expand Up @@ -523,6 +524,7 @@ def _iter_eligible_file_paths(
exclude_files: Sequence[str],
exclude_folders: Sequence[str],
output_file_abs: str,
temp_file_abs: str | None = None,
is_cancelled: Callable[[], bool] | None = None,
) -> Iterable[str]:
"""Yield eligible file paths while respecting cancellation hooks."""
Expand Down Expand Up @@ -571,7 +573,17 @@ def _iter_eligible_file_paths(
)
]

dirs[:] = filtered_dirs
# Stable ordering ensures preview and extraction produce the same plan.
dirs[:] = sorted(filtered_dirs)
filtered_files.sort(
key=lambda name: (
0
if os.path.abspath(root) == os.path.abspath(folder_path)
and name in SPECIFICATION_FILES
else 1,
name,
)
)

for file_name in filtered_files:
if is_cancelled and is_cancelled():
Expand All @@ -580,7 +592,10 @@ def _iter_eligible_file_paths(
)

file_path = os.path.join(root, file_name)
if os.path.abspath(file_path) == output_file_abs:
# File symlinks can otherwise expose data outside the selected tree.
if os.path.islink(file_path):
continue
if os.path.abspath(file_path) in (output_file_abs, temp_file_abs):
continue
if file_path in self.processed_files or file_path in seen_paths:
continue
Expand Down Expand Up @@ -640,17 +655,23 @@ def extract_files(
if mode_normalised == "inclusion" and not normalised_extensions:
normalised_extensions = tuple(COMMON_EXTENSIONS)

temp_output_path: str | None = None
try:
output_file_abs = os.path.abspath(output_file_name)

with open(output_file_name, "w", encoding="utf-8") as output:
skipped_specs = self.process_specifications(
folder_path,
output,
is_cancelled=is_cancelled,
)
skipped_count += skipped_specs

# Stage output next to its destination for atomic replacement.
# Incomplete/failed runs preserve the previous output.
with tempfile.NamedTemporaryFile(
mode="w",
encoding="utf-8",
dir=os.path.dirname(output_file_abs),
prefix=".file-extractor-",
suffix=".tmp",
delete=False,
) as output:
temp_output_path = output.name
# All files, including README/specifications, must pass the same
# inclusion/exclusion filter. Prioritisation lives in traversal.
# Fix: Q-102 - pre-compute eligible files but fall back when memory is scarce.
total_files: int
iteration_source: Iterable[str]
Expand All @@ -664,6 +685,7 @@ def extract_files(
exclude_files=exclude_files,
exclude_folders=exclude_folders,
output_file_abs=output_file_abs,
temp_file_abs=temp_output_path,
is_cancelled=is_cancelled,
)
)
Expand All @@ -685,6 +707,7 @@ def extract_files(
exclude_files=exclude_files,
exclude_folders=exclude_folders,
output_file_abs=output_file_abs,
temp_file_abs=temp_output_path,
is_cancelled=is_cancelled,
)
else:
Expand Down Expand Up @@ -721,15 +744,18 @@ def extract_files(
if progress_callback:
progress_callback(processed_count, total_files)

completion_message = (
f"Extraction complete. Processed {processed_count} files. "
f"Results written to {output_file_name}."
# The complete staged output is now safely closed and committed.
os.replace(temp_output_path, output_file_abs)
temp_output_path = None
completion_message = (
f"Extraction complete. Processed {processed_count} files. "
f"Results written to {output_file_name}."
)
if skipped_count:
completion_message += (
f" Skipped {skipped_count} files due to safeguards."
)
if skipped_count:
completion_message += (
f" Skipped {skipped_count} files due to safeguards."
)
self._enqueue_message("info", completion_message)
self._enqueue_message("info", completion_message)

except Exception as exc:
error_msg = f"Error during extraction: {exc}"
Expand Down Expand Up @@ -800,6 +826,12 @@ def extract_files(
f"skipped={skipped_count}"
),
)
finally:
if temp_output_path is not None:
try:
os.unlink(temp_output_path)
except FileNotFoundError:
pass


__all__ = [
Expand Down
34 changes: 34 additions & 0 deletions services/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,8 @@ class CLIOptions:
max_file_size_mb: int | None = None
poll_interval: float = DEFAULT_POLL_INTERVAL
log_level: str = "INFO"
dry_run: bool = False
preview_format: str = "text"


# Fix: Q-101 - support nested sequences from repeated --extensions flags.
Expand Down Expand Up @@ -155,6 +157,17 @@ def build_parser() -> argparse.ArgumentParser:
default=(),
help="Folder name patterns to exclude",
)
parser.add_argument(
"--dry-run",
action="store_true",
help="Show the files that would be extracted without writing anything",
)
parser.add_argument(
"--preview-format",
choices=("text", "json"),
default="text",
help="Output format for --dry-run (default: text)",
)
parser.add_argument(
"--output",
type=Path,
Expand Down Expand Up @@ -217,6 +230,8 @@ def parse_arguments(argv: Sequence[str] | None = None) -> CLIOptions:
max_file_size_mb=args.max_file_size_mb,
poll_interval=args.poll_interval,
log_level=args.log_level,
dry_run=args.dry_run,
preview_format=args.preview_format,
)


Expand Down Expand Up @@ -285,6 +300,25 @@ def run_cli(
) -> int:
"""Execute the extraction using the service layer and return an exit code."""

if options.dry_run:
from services.preview import build_preview, format_preview

try:
result = build_preview(
folder_path=options.folder_path,
mode=options.mode,
include_hidden=options.include_hidden,
extensions=options.extensions,
exclude_files=options.exclude_files,
exclude_folders=options.exclude_folders,
output_file=options.output_file,
)
except (OSError, ValueError) as exc:
print(f"Preview failed: {exc}", file=sys.stderr)
return 1
print(format_preview(result, options.preview_format))
return 0

if configure_logger_handler and not logger.handlers:
stream_handler = logging.StreamHandler(stream=sys.stdout)
configure_logging(
Expand Down
112 changes: 112 additions & 0 deletions services/preview.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,112 @@
"""Read-only, deterministic preview of a proposed extraction run.

Both preview and extraction use FileProcessor._iter_eligible_file_paths so that
there are no independent, diverging file-selection implementations.
"""

from __future__ import annotations

import json
import os
from pathlib import Path
from queue import Queue
from typing import Any, Sequence

from constants import COMMON_EXTENSIONS
from processor import FileProcessor


def build_preview(
*,
folder_path: Path,
mode: str,
include_hidden: bool,
extensions: Sequence[str],
exclude_files: Sequence[str],
exclude_folders: Sequence[str],
output_file: Path,
) -> dict[str, Any]:
"""Produce an exact file-selection plan without creating or modifying files."""
root = folder_path.resolve(strict=True)
if not root.is_dir():
raise ValueError(f"Not a directory: {root}")

normalized_mode = mode.lower()
if normalized_mode not in {"inclusion", "exclusion"}:
raise ValueError(f"Unsupported extraction mode: {mode}")

selected_extensions = (
tuple(COMMON_EXTENSIONS)
if normalized_mode == "inclusion" and not extensions
else tuple(extensions)
)
processor = FileProcessor(Queue())
entries: list[dict[str, Any]] = []
warnings: list[str] = []
output_abs = os.path.abspath(output_file)

for file_path in processor._iter_eligible_file_paths(
folder_path=str(root),
mode=normalized_mode,
include_hidden=include_hidden,
extensions=selected_extensions,
exclude_files=exclude_files,
exclude_folders=exclude_folders,
output_file_abs=output_abs,
):
try:
size = os.path.getsize(file_path)
except OSError as exc:
warnings.append(f"Could not stat {file_path}: {exc}")
continue
entries.append(
{
"path": Path(file_path).relative_to(root).as_posix(),
"size_bytes": size,
}
)

if output_file.exists():
warnings.append(f"Extraction will overwrite existing output: {output_file}")

total_bytes = sum(entry["size_bytes"] for entry in entries)
return {
"source": str(root),
"output": str(output_file),
"mode": normalized_mode,
"file_count": len(entries),
"total_bytes": total_bytes,
"files": entries,
"warnings": warnings,
"note": (
"Preview reports selection and source sizes; unreadable or non-UTF-8 "
"files may be skipped during extraction."
),
}


def format_preview(preview: dict[str, Any], format_name: str = "text") -> str:
"""Render the preview as readable terminal text or machine-readable JSON."""
if format_name == "json":
return json.dumps(preview, indent=2, ensure_ascii=False)
if format_name != "text":
raise ValueError(f"Unsupported preview format: {format_name}")

lines = [
"File Extractor Pro — read-only preview",
f"Source: {preview['source']}",
f"Output: {preview['output']}",
f"Selected: {preview['file_count']} files, "
f"{preview['total_bytes']} source bytes",
"",
]
for entry in preview["files"]:
lines.append(f" {entry['path']} ({entry['size_bytes']} bytes)")
if preview["warnings"]:
lines.extend(["", "Warnings:"])
lines.extend(f" - {warning}" for warning in preview["warnings"])
lines.extend(["", str(preview["note"])])
return "\n".join(lines)


__all__ = ["build_preview", "format_preview"]
Loading
Loading