diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..0aa4c34 --- /dev/null +++ b/.flake8 @@ -0,0 +1,18 @@ +[flake8] +max-line-length = 100 +max-complexity = 35 +exclude = + .git, + __pycache__, + .venv, + venv, + build, + dist, + baseline +extend-ignore = + E203, + E501, + W503 +per-file-ignores = + lib/pyseq/__init__.py:F401,F403 + tests/test_pyseq.py:E402 diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml new file mode 100644 index 0000000..65eb29b --- /dev/null +++ b/.github/workflows/benchmarks.yml @@ -0,0 +1,244 @@ +name: benchmarks + +on: + pull_request: + types: + - opened + - synchronize + - reopened + - ready_for_review + - converted_to_draft + paths: + - "lib/pyseq/**" + - "scripts/benchmark.py" + - ".github/workflows/benchmarks.yml" + workflow_dispatch: + inputs: + profile: + description: Benchmark profile to run + required: false + default: smoke + type: choice + options: + - smoke + - full + schedule: + - cron: "0 9 * * 1" + +permissions: + contents: read + pull-requests: write + +jobs: + benchmark-pr: + if: github.event_name == 'pull_request' + runs-on: ubuntu-24.04 + + steps: + - name: Check out candidate branch + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Install package dependencies + run: | + python -m pip install --upgrade pip setuptools wheel + python -m pip install -e ".[dev]" + + - name: Check out baseline branch + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.base.sha }} + path: baseline + + - name: Run baseline benchmarks + working-directory: baseline + run: | + mkdir -p scripts + cp "$GITHUB_WORKSPACE/scripts/benchmark.py" scripts/benchmark.py + python scripts/benchmark.py --scenarios contiguous,mixed --json baseline-benchmark.json > baseline-benchmark.md + + - name: Run candidate benchmarks + run: | + python scripts/benchmark.py --scenarios contiguous,mixed --json candidate-benchmark.json > candidate-benchmark.md + + - name: Compare benchmark runs + id: compare + if: success() || hashFiles('baseline/baseline-benchmark.json', 'candidate-benchmark.json') != '' + run: | + set +e + python scripts/benchmark.py \ + --compare baseline/baseline-benchmark.json candidate-benchmark.json \ + --json benchmark-compare.json \ + --fail-on-regression-pct 3 \ + --fail-on-regression-s 0.005 > benchmark-compare.md + status=$? + echo "exit_code=$status" >> "$GITHUB_OUTPUT" + if [ "$status" -eq 0 ]; then + echo "threshold_exceeded=false" >> "$GITHUB_OUTPUT" + else + echo "threshold_exceeded=true" >> "$GITHUB_OUTPUT" + fi + exit 0 + + - name: Write benchmark summaries + if: always() + shell: bash + run: | + { + if [ -f candidate-benchmark.md ]; then + echo "## Candidate benchmark" + cat candidate-benchmark.md + echo + fi + if [ -f baseline/baseline-benchmark.md ]; then + echo "## Baseline benchmark" + cat baseline/baseline-benchmark.md + echo + fi + if [ -f benchmark-compare.md ]; then + echo "## Comparison" + cat benchmark-compare.md + else + echo "## Comparison" + echo "Benchmark comparison did not run to completion." + fi + } >> "$GITHUB_STEP_SUMMARY" + + - name: Post pull request benchmark comment + if: always() && hashFiles('benchmark-compare.md') != '' + uses: actions/github-script@v7 + env: + BENCHMARK_COMPARE_PATH: benchmark-compare.md + BENCHMARK_THRESHOLD_EXCEEDED: ${{ steps.compare.outputs.threshold_exceeded }} + with: + script: | + const fs = require("fs"); + const marker = ""; + const compare = fs.readFileSync(process.env.BENCHMARK_COMPARE_PATH, "utf8"); + const failed = process.env.BENCHMARK_THRESHOLD_EXCEEDED === "true"; + const header = failed + ? "## Benchmark Regression Alert" + : "## Benchmark Comparison"; + const status = failed + ? "_Regression guard exceeded the +3.00% and +0.005s thresholds._" + : "_No benchmark exceeded the +3.00% and +0.005s regression thresholds._"; + const body = `${marker}\n${header}\n\n${status}\n\n${compare}`; + + const { owner, repo } = context.repo; + const issue_number = context.issue.number; + const comments = await github.paginate(github.rest.issues.listComments, { + owner, + repo, + issue_number, + per_page: 100, + }); + + const existing = comments.find((comment) => comment.body.includes(marker)); + if (existing) { + await github.rest.issues.updateComment({ + owner, + repo, + comment_id: existing.id, + body, + }); + } else { + await github.rest.issues.createComment({ + owner, + repo, + issue_number, + body, + }); + } + + - name: Upload benchmark artifacts + if: always() + shell: bash + run: | + mkdir -p benchmark-artifacts/baseline + if [ -f baseline/baseline-benchmark.json ]; then + cp baseline/baseline-benchmark.json benchmark-artifacts/baseline/ + fi + if [ -f baseline/baseline-benchmark.md ]; then + cp baseline/baseline-benchmark.md benchmark-artifacts/baseline/ + fi + if [ -d baseline/tmp/benchmarks/profiles ]; then + mkdir -p benchmark-artifacts/baseline/tmp/benchmarks + cp -R baseline/tmp/benchmarks/profiles benchmark-artifacts/baseline/tmp/benchmarks/ + fi + + - name: Publish benchmark artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: benchmark-results-pr-${{ github.event.pull_request.number }} + path: | + candidate-benchmark.json + candidate-benchmark.md + benchmark-compare.json + benchmark-compare.md + tmp/benchmarks/profiles + benchmark-artifacts/baseline + if-no-files-found: warn + + - name: Enforce regression threshold + if: always() && steps.compare.outputs.threshold_exceeded == 'true' + run: | + echo "Benchmark regression exceeded the +3.00% and +0.005s thresholds." + exit 1 + + benchmark-branch: + if: github.event_name != 'pull_request' + runs-on: ubuntu-24.04 + + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Install package + run: | + python -m pip install --upgrade pip setuptools wheel + python -m pip install -e ".[dev]" + + - name: Select profile + id: profile + shell: bash + run: | + if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then + echo "value=${{ inputs.profile }}" >> "$GITHUB_OUTPUT" + elif [ "${{ github.event_name }}" = "schedule" ]; then + echo "value=full" >> "$GITHUB_OUTPUT" + else + echo "value=smoke" >> "$GITHUB_OUTPUT" + fi + + - name: Run benchmarks + run: | + if [ "${{ steps.profile.outputs.value }}" = "full" ]; then + python scripts/benchmark.py --full --json benchmark.json > benchmark.md + else + python scripts/benchmark.py --json benchmark.json > benchmark.md + fi + + - name: Write benchmark summary + run: | + cat benchmark.md >> "$GITHUB_STEP_SUMMARY" + + - name: Upload benchmark artifacts + uses: actions/upload-artifact@v4 + with: + name: benchmark-results-${{ steps.profile.outputs.value }} + path: | + benchmark.json + benchmark.md + tmp/benchmarks/profiles diff --git a/.github/workflows/publish-docs.yml b/.github/workflows/publish-docs.yml new file mode 100644 index 0000000..aeaf1f0 --- /dev/null +++ b/.github/workflows/publish-docs.yml @@ -0,0 +1,72 @@ +name: publish-docs + +on: + push: + branches: + - master + - main + workflow_dispatch: + +permissions: + contents: read + pages: write + id-token: write + +concurrency: + group: pages + cancel-in-progress: true + +jobs: + build: + runs-on: ubuntu-24.04 + + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Configure Pages + uses: actions/configure-pages@v5 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Install package + run: | + python -m pip install --upgrade pip setuptools wheel + python -m pip install -e ".[dev]" + + - name: Generate benchmark report + run: | + python scripts/benchmark.py --json benchmark.json > benchmark.md + + - name: Build Jekyll source tree + run: | + python scripts/build_pages_site.py \ + --output _site_src \ + --benchmark-summary benchmark.md \ + --benchmark-json benchmark.json + + - name: Build site with Jekyll + uses: actions/jekyll-build-pages@v1 + with: + source: _site_src + destination: _site + + - name: Upload Pages artifact + uses: actions/upload-pages-artifact@v3 + with: + path: _site + + deploy: + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + runs-on: ubuntu-24.04 + needs: build + + steps: + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v4 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 6567195..015f6c3 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -4,10 +4,50 @@ on: push: branches: - main + paths: + - "lib/pyseq/**" + - "tests/**" + - "scripts/**" + - "pyproject.toml" + - ".flake8" + - ".github/workflows/tests.yml" pull_request: + paths: + - "lib/pyseq/**" + - "tests/**" + - "scripts/**" + - "pyproject.toml" + - ".flake8" + - ".github/workflows/tests.yml" workflow_dispatch: jobs: + quality: + runs-on: ubuntu-latest + + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Install package and quality dependencies + run: | + python -m pip install --upgrade pip setuptools wheel + python -m pip install -e ".[dev]" + + - name: Check import ordering + run: python -m isort --check-only --diff lib/pyseq tests scripts + + - name: Check black formatting + run: python -m black --check lib/pyseq tests scripts + + - name: Run flake8 and mccabe checks + run: python -m flake8 lib/pyseq tests scripts + test: runs-on: ${{ matrix.os }} strategy: diff --git a/.gitignore b/.gitignore index ec64e06..e2de1e8 100644 --- a/.gitignore +++ b/.gitignore @@ -11,4 +11,7 @@ tmp/ venv*/ .vscode/ pyseq.egg-info/ -default.env \ No newline at end of file +default.env +.codex +.agents/ +.pytest_cache/ \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index b2e9656..1cde449 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,17 @@ CHANGELOG ========= +## 0.9.3 + +* Adds canonical stepped frame range formatting via `%x`, using compact syntax such as `1-10x2,20-30x5,42` +* Adds explicit stepped range parsing for serialized and embedded sequence references +* Adds support for signed frame numbers in serialized ranges and on-disk filenames such as `render.-0002.exr` +* Refactors explicit range parsing and formatting onto shared helpers to keep the syntax rules consistent +* Expands regression coverage for stepped ranges, negative frame handling, CLI tools, and performance-sensitive paths +* Adds benchmark tooling, benchmark documentation, and pull request regression checks for performance-sensitive changes +* Sets CI benchmark regression thresholds to `3%` and `0.005s`, while still publishing full benchmark deltas in pull request comments +* Reduces `lss` startup overhead by deferring non-critical imports and moving extended frame-range regex compilation off the default import path + ## 0.9.2 * Renames the move CLI from `smove` to `smv` diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..e93cdcd --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,51 @@ +# Contributing + +PySeq changes should stay small, clear, and performance-aware. + +## Development Basics + +- Run the unit tests for code changes: + `pytest tests -q` +- Run local quality checks for Python changes: + `python -m black --check lib/pyseq tests scripts` + `python -m isort --check-only --diff lib/pyseq tests scripts` + `python -m flake8 lib/pyseq tests scripts` +- Run those commands from the repo's `.venv` so local results match CI. +- Prefer focused changes over broad refactors unless the refactor is the task. +- Keep the default filename discovery path permissive and fast. + +Optional local hook installation: + +- `python scripts/install_precommit_hook.py` + +That hook runs `black`, `isort`, `flake8`/`mccabe`, and `pytest` before each +commit. + +## Performance + +Performance is a project priority, especially for: + +- `lss` +- `get_sequences(...)` +- `resolve_sequence(...)` + +When changing code under `lib/pyseq/`, contributors should treat performance +as part of correctness. + +Current benchmark policy: + +- pull requests run synthetic benchmark comparisons against the base branch +- benchmark deltas stay visible in the pull request comment +- the benchmark workflow fails when a regression exceeds both `3%` and `0.005s` +- repeated small regressions in the same hotspot should be investigated before + they accumulate across releases + +Recommended workflow for performance-sensitive changes: + +1. Run the unit tests. +2. Run `python scripts/benchmark.py --json /tmp/pyseq-feature.json`. +3. Compare against a baseline branch or commit with + `python scripts/benchmark.py --compare /tmp/pyseq-main.json /tmp/pyseq-feature.json`. +4. If a hotspot regresses, collect profiles and inspect them before merging. + +For more detail, see [docs/performance.md](docs/performance.md). diff --git a/README.md b/README.md index 6bdfc7b..c50a255 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,7 @@ Used in visual effects, animation, and post-production pipelines. [Production Usage](#production-usage) | [Command-Line Tools](#command-line-tools) | [Docs](#docs) | +[Contributing](#contributing) | [Testing](#testing) ## Installation @@ -87,6 +88,32 @@ Deserialize a compressed sequence string: 1001 012_vb_110_v001.%04d.png [1-1001] ``` +Extended serialized range syntax is also supported for uncompression and +sequence reference parsing: + +```python +>>> s = uncompress('render.%04d.exr 1001-1010x3', fmt='%h%p%t %x') +>>> print(s.frames()) +[1001, 1004, 1007, 1010] +>>> print(s.format('%h%p%t %x')) +render.%04d.exr 1001-1010x3 +``` + +Supported serialized range forms include: + +- `1001-1100` +- `1001-1100x2` +- `1001-1100x10, 1200, 1300-1320x5` +- `10-1x3` +- `-10--1x3` with `PYSEQ_ALLOW_NEGATIVE_FRAMES=1` + +Negative frame ranges are opt-in so the default filename discovery path stays +fast and unopinionated. Enable them with: + +```bash +export PYSEQ_ALLOW_NEGATIVE_FRAMES=1 +``` + ## Production Usage pyseq has been used for many years in production visual effects and animation @@ -119,6 +146,10 @@ Additional documentation is available in the [docs](docs/) folder: - [Formatting Reference](docs/formatting.md) - [Frame Patterns](docs/frame-patterns.md) +## Contributing + +Contributor guidance lives in [CONTRIBUTING.md](CONTRIBUTING.md). + ## Testing To run the unit tests, simply run `pytest` in a shell: diff --git a/docs/README.md b/docs/README.md index 513d9e8..ed6287a 100644 --- a/docs/README.md +++ b/docs/README.md @@ -10,6 +10,7 @@ Available guides: - [Setup and Distribution](setup-and-distribution.md) - [Formatting Reference](formatting.md) - [Frame Patterns](frame-patterns.md) +- [Performance](performance.md) If you are evaluating pyseq for pipeline use, start with the examples guide and then review the CLI reference for the sequence-aware utilities included with diff --git a/docs/cli-tools.md b/docs/cli-tools.md index 60fc827..3ceba89 100644 --- a/docs/cli-tools.md +++ b/docs/cli-tools.md @@ -11,6 +11,7 @@ List files and group matching names into sequences. lss tests/files lss -r tests lss tests/files/*.png -f "%l %h%r%t %M" +lss tests/files/*.png -f "%h%p%t %x" ``` ## `stree` @@ -54,6 +55,7 @@ sequence. ```bash scopy input.%04d.exr output/ scopy input.1-100.exr scene.1001-1100.exr +scopy input.%04d.exr\ 1001-1010x2 output/ ``` ## `smv` @@ -63,6 +65,7 @@ Move or rename a sequence, with optional renumbering. ```bash smv old.%04d.exr new.%04d.exr smv old.%04d.exr archive/ --renumber 1001 +smv old.%04d.exr\ 1001-1005x2 new.%04d.exr\ 2001-2003 ``` ## `srm` @@ -71,6 +74,7 @@ Remove a sequence or a frame range embedded in a compressed sequence string. ```bash srm input.1-100.exr +srm input.%04d.exr\ 1001-1010x2 ``` ## Common Pipeline Use Cases diff --git a/docs/examples.md b/docs/examples.md index 7af1e70..98fd80a 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -99,6 +99,53 @@ comp.1001.exr comp.1005.exr ``` +## Parse a Stepped Sequence String + +```python +from pyseq import uncompress + +seq = uncompress("render.%04d.exr 1001-1010x3", fmt="%h%p%t %x") + +print(seq.frames()) +print(seq.format("%h%p%t %x")) +``` + +Expected output: + +```text +[1001, 1004, 1007, 1010] +render.%04d.exr 1001-1010x3 +``` + +## Parse Signed Serialized Frame Ranges + +Signed frame ranges are opt-in. Enable them first: + +```bash +export PYSEQ_ALLOW_NEGATIVE_FRAMES=1 +``` + +Then serialized range strings may include signed frame numbers: + +```python +from pyseq import uncompress + +seq = uncompress("render.%04d.exr -10--1x3", fmt="%h%p%t %x") + +print(seq.frames()) +print(seq.format("%x")) +``` + +Expected output: + +```text +[-10, -7, -4, -1] +-10--1x3 +``` + +Signed frame ranges also work with on-disk filename patterns such as +`render.-0010.exr` when resolving or discovering sequences. + ## Resolve a Sequence Pattern on Disk `resolve_sequence` looks up files on disk from a compressed sequence pattern and diff --git a/docs/formatting.md b/docs/formatting.md index afe72b3..ffb1914 100644 --- a/docs/formatting.md +++ b/docs/formatting.md @@ -16,12 +16,38 @@ CLI tools. | `%p` | padding, e.g. %06d | | `%r` | implied range, start-end | | `%R` | explicit broken range, [1-10, 15-20] | +| `%x` | stepped explicit range, 1-10x2,20 | | `%d` | disk usage | | `%H` | disk usage (human readable) | | `%D` | parent directory | | `%h` | string preceding sequence number | | `%t` | string after the sequence number | +## Serialized Range Syntax + +PySeq supports the following serialized range forms when parsing sequence +strings with `uncompress()` and the sequence-aware CLI tools: + +```text +1001 +1001-1100 +1001-1100x2 +1001-1100x10, 1200, 1300-1320x5 +10-1x3 +-10--1x3 +``` + +Notes: + +- `xN` means "every Nth frame" anchored to the segment start. +- Descending ranges are accepted when parsing, but `Sequence.frames()` and + `%x` formatting normalize frames into ascending order. +- `%x` uses compact canonical formatting with no spaces after commas. +- Negative frame ranges are opt-in. Enable them with + `PYSEQ_ALLOW_NEGATIVE_FRAMES=1`. +- With that flag enabled, signed frame numbers are supported in serialized + range strings and in on-disk filename patterns such as `render.-0010.exr`. + ## CLI Examples ```bash @@ -33,6 +59,9 @@ $ lss tests/files/a*.tga -f "%l %h%r%t" $ lss tests/files/a*.tga -f "%l %h%r%t %M" 7 a.1-14.tga [4-9, 11] + +$ python3 -c "import pyseq; s = pyseq.uncompress('render.%04d.exr 1001-1010x3', fmt='%h%p%t %x'); print(s.format('%x'))" +1001-1010x3 ``` ## Python Examples @@ -45,6 +74,7 @@ seq = pyseq.get_sequences("tests/files/a*.tga")[0] print(seq.format("%h%r%t")) print(seq.format("%l %h%r%t")) print(seq.format("%l %h%r%t %M")) +print(pyseq.uncompress("render.%04d.exr 1001-1010x3", fmt="%h%p%t %x").format("%x")) ``` Expected output: @@ -53,4 +83,5 @@ Expected output: a.1-14.tga 7 a.1-14.tga 7 a.1-14.tga [4-9, 11] +1001-1010x3 ``` diff --git a/docs/frame-patterns.md b/docs/frame-patterns.md index 3fd333a..90be590 100644 --- a/docs/frame-patterns.md +++ b/docs/frame-patterns.md @@ -3,6 +3,12 @@ Use `${PYSEQ_FRAME_PATTERN}` to define custom regular expressions for identifying frame numbers. +Signed frame numbers such as `render.-0010.exr` are opt-in. Enable them with: + +```bash +export PYSEQ_ALLOW_NEGATIVE_FRAMES=1 +``` + ## Example If frames are always preceded by an underscore: @@ -35,4 +41,7 @@ PYSEQ_FRAME_PATTERN: \.\d+\. # frame numbers start with an underscore, e.g. file_v1_1001.exr PYSEQ_FRAME_PATTERN: _\d+ + +# allow signed frame numbers in explicit ranges and filename discovery +PYSEQ_ALLOW_NEGATIVE_FRAMES: ${PYSEQ_ALLOW_NEGATIVE_FRAMES:=0} ``` diff --git a/docs/performance.md b/docs/performance.md new file mode 100644 index 0000000..524f1a9 --- /dev/null +++ b/docs/performance.md @@ -0,0 +1,262 @@ +# Performance + +PySeq's performance matters most in sequence discovery and `lss` over large +file sets. + +This project uses two complementary approaches: + +- coarse regression tests in `tests/test_performance.py` +- one repeatable benchmark runner in `scripts/benchmark.py` + +The regression tests should stay stable and fast enough for normal CI. The +benchmark runner is for collecting timing and profiling data over time, not +for enforcing fragile micro-optimizations in every pull request. + +## Benchmark Runner + +Current benchmark entry point: + +- `scripts/benchmark.py` + +Simple examples: + +```bash +python scripts/benchmark.py +python scripts/benchmark.py --full +python scripts/benchmark.py --json /tmp/pyseq-main.json +python scripts/benchmark.py --scenarios contiguous,mixed +python scripts/benchmark.py --path /project/shot/frames --iterations 5 +python scripts/benchmark.py --path /project/shot/frames --glob "*.exr" --iterations 5 +python scripts/benchmark.py --compare /tmp/pyseq-main.json /tmp/pyseq-feature.json +``` + +What it measures: + +- `get_sequences(...)` from an in-memory file list +- `get_sequences(...)` from a directory path +- `resolve_sequence(...)` against a padded on-disk sequence +- `lss` end-to-end subprocess runtime on the same synthetic dataset + +Synthetic scenarios: + +- `contiguous`: one large primary sequence with light non-sequence noise +- `mixed`: a more realistic directory with a dominant gapped sequence, + several neighboring sequences, and unrelated files + +When using `--path`, the runner benchmarks an existing directory instead of a +synthetic dataset. This is useful for validating results against real-world +sequence layouts that may not be modeled well by generated fixtures. + +Default synthetic dataset sizes: + +- default: `100`, `1000`, `10000` +- `--full`: `100`, `1000`, `10000`, `50000` + +Default synthetic scenarios: + +- `contiguous` +- `mixed` + +You can override dataset sizes or iteration count: + +```bash +python scripts/benchmark.py --sizes 100,1000,10000,50000 --iterations 7 +python scripts/benchmark.py --sizes 1000,10000 --scenarios mixed --iterations 7 +``` + +## Local A/B Comparison + +Use the same machine and same interpreter for both runs. + +Example workflow: + +```bash +# on baseline branch +python scripts/benchmark.py --full --json /tmp/pyseq-main.json + +# switch branches +python scripts/benchmark.py --full --json /tmp/pyseq-feature.json + +# compare +python scripts/benchmark.py --compare /tmp/pyseq-main.json /tmp/pyseq-feature.json +``` + +For real-world validation, use the same workflow against an existing directory: + +```bash +# on baseline branch +python scripts/benchmark.py --path /project/shot/frames --iterations 5 --json /tmp/pyseq-main.json + +# switch branches +python scripts/benchmark.py --path /project/shot/frames --iterations 5 --json /tmp/pyseq-feature.json + +# compare +python scripts/benchmark.py --compare /tmp/pyseq-main.json /tmp/pyseq-feature.json +``` + +## Benchmark Hygiene + +Before trusting an A/B result, make sure both runs use: + +- the same terminal/session state +- the same Python interpreter or virtualenv +- the same benchmark script revision +- the same target directory or synthetic dataset shape + +The benchmark summary and JSON now record: + +- `python_executable` +- `lib_root` +- `pyseq_file` +- `pyseq_seq_file` + +Check those fields before drawing conclusions from a comparison. If they do +not match your expectations, fix the environment first and rerun. + +## Profiling + +Profiling is enabled by default, but it runs in a separate pass after the +timed measurements. That means the timing samples stay clean while we still +capture where time is going. + +Generated artifacts: + +- one `.pstats` file per benchmark case +- one text summary per benchmark case, sorted by cumulative time + +Default output location: + +- `tmp/benchmarks/profiles/` + +Useful options: + +- `--no-profile` to skip profiling entirely +- `--profiles-dir ` to choose a different output directory +- `--scenarios ` to choose synthetic directory shapes +- `--fail-on-regression-pct ` to turn a compare run into a CI gate + +## CI Automation + +The GitHub Actions benchmark workflow runs pull request comparisons in one job +on one GitHub-hosted runner. + +For pull requests it: + +- checks out the candidate branch +- checks out the pull request base commit in a separate `baseline/` tree +- runs the same benchmark script revision against both trees +- compares the two JSON result sets +- uploads both result sets and profile artifacts +- posts a sticky pull request comment with the comparison summary +- fails the workflow if any benchmark regresses by more than `3%` and `0.005s` + +On pull requests, `benchmark-pr` is the authoritative check. The separate +`benchmark-branch` job is only for non-PR runs such as scheduled or manual +benchmark collection, so seeing it skipped on a pull request is expected. + +The pull request suite uses synthetic datasets only, so it is best treated as +a regression guardrail. For more realistic validation, keep using local +`--path` runs against representative production-style directories. + +Current benchmark policy: + +- keep every benchmark delta visible in the PR comment +- fail CI only when a regression exceeds both `3%` and `0.005s` +- treat repeated small regressions in the same hotspot as a reason to profile + and optimize before they accumulate over time + +## Result Format + +The benchmark runner can emit machine-readable JSON for local comparison, CI +artifacts, or published reports. + +Example run payload: + +```json +{ + "benchmarks": { + "get_sequences_dir_1000": { + "iterations": 5, + "max_s": 0.043512, + "mean_s": 0.041932, + "median_s": 0.041704, + "min_s": 0.040161, + "samples_s": [0.040161, 0.041085, 0.041704, 0.043198, 0.043512] + }, + "lss_1000": { + "iterations": 5, + "max_s": 0.139424, + "mean_s": 0.133529, + "median_s": 0.132611, + "min_s": 0.129400, + "samples_s": [0.129400, 0.131818, 0.132611, 0.134390, 0.139424] + } + }, + "branch": "feat/example", + "commit": "abc1234", + "iterations": 5, + "kind": "benchmark", + "lib_root": "/path/to/pyseq/lib", + "platform": "Linux-6.x-x86_64", + "profiled": true, + "pyseq_file": "/path/to/pyseq/lib/pyseq/__init__.py", + "pyseq_seq_file": "/path/to/pyseq/lib/pyseq/seq.py", + "python_executable": "/path/to/venv/bin/python", + "profiles_dir": "tmp/benchmarks/profiles/20260721T000000Z-feat-example-abc1234", + "python": "3.11.x", + "repo_root": "/path/to/pyseq", + "scenarios": ["contiguous", "mixed"], + "script_path": "/path/to/pyseq/scripts/benchmark.py", + "sizes": [100, 1000, 10000], + "timestamp_utc": "2026-07-21T00:00:00+00:00" +} +``` + +Field notes: + +- `kind` distinguishes normal benchmark runs from comparison payloads. +- `median_s` is the primary comparison value. +- `mean_s` is useful when the run-to-run spread is small. +- `samples_s` helps show spread directly. +- `branch` and `commit` make branch-to-branch comparisons easier to track. +- `repo_root`, `lib_root`, `script_path`, and `python_executable` show + exactly which source tree and interpreter were used. +- `scenarios` records which synthetic directory shapes were exercised. +- `pyseq_file` and `pyseq_seq_file` confirm which imported package files were + actually exercised. +- `profiles_dir` points to the profile output location for that run. +- `timestamp_utc`, `python`, and `platform` help when comparing results. + +## Methodology + +The benchmark runner: + +- generates synthetic datasets in temporary directories +- uses `time.perf_counter()` +- warms up once before measuring +- collects several samples +- reports median, mean, min, and max +- profiles each case in a separate pass by default + +This keeps the results useful without pretending that a shared CI runner is a +perfect benchmarking machine. + +## Interpreting Results + +Treat GitHub-hosted benchmark numbers as trend indicators, not as precise +absolute truth. + +Good uses: + +- spotting real regressions on the same machine +- comparing branch-to-branch trends on the same interpreter +- attaching timing evidence to performance-focused pull requests + +Less reliable uses: + +- failing builds on tiny percentage differences +- comparing results across different operating systems or Python versions +- drawing conclusions from a single run + +If stronger consistency becomes important, prefer a self-hosted benchmark +runner with a stable machine configuration. diff --git a/lib/pyseq/__init__.py b/lib/pyseq/__init__.py index 7800c27..c5aec67 100644 --- a/lib/pyseq/__init__.py +++ b/lib/pyseq/__init__.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Copyright (c) 2011-2025, Ryan Galloway (ryan@rsgalloway.com) +# Copyright (c) 2011-2026, Ryan Galloway (ryan@rsgalloway.com) # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: @@ -48,6 +48,6 @@ """ __author__ = "Ryan Galloway" -__version__ = "0.9.2" +__version__ = "0.9.3" from .seq import * diff --git a/lib/pyseq/cli.py b/lib/pyseq/cli.py new file mode 100644 index 0000000..85389f0 --- /dev/null +++ b/lib/pyseq/cli.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python +# +# Copyright (c) 2011-2026, Ryan Galloway (ryan@rsgalloway.com) +# + +"""Lightweight helpers shared by CLI entry points.""" + +import functools +import sys + + +def cli_catch_keyboard_interrupt(func): + """Return exit code 1 instead of a traceback on Ctrl-C.""" + + @functools.wraps(func) + def inner(*args, **kwargs): + try: + return func(*args, **kwargs) + except KeyboardInterrupt: + print("stopping...", file=sys.stderr) + return 1 + + return inner diff --git a/lib/pyseq/config.py b/lib/pyseq/config.py index 21e4a32..d1ab583 100644 --- a/lib/pyseq/config.py +++ b/lib/pyseq/config.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Copyright (c) 2011-2025, Ryan Galloway (ryan@rsgalloway.com) +# Copyright (c) 2011-2026, Ryan Galloway (ryan@rsgalloway.com) # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: @@ -49,13 +49,38 @@ PYSEQ_NOT_STRICT = os.getenv("PYSEQ_NOT_STRICT", 1) strict_pad = int(PYSEQ_STRICT_PAD) == 1 or int(PYSEQ_NOT_STRICT) == 0 -# regex pattern for matching all numbers in a filename -digits_re = re.compile(r"\d+") - # regex pattern for matching frame numbers only -# the default is \d+ for maximum compatibility +# pyseq intentionally stays permissive here and lets sibling/diff logic decide +# whether matching numeric tokens represent the sequence frame component. DEFAULT_FRAME_PATTERN = r"\d+" +DEFAULT_SIGNED_FRAME_PATTERN = r"(?:(?<=^)|(?<=[._]))-\d+|\d+" + +# regex pattern for matching all numeric sequence tokens in a filename +digits_re = re.compile(DEFAULT_FRAME_PATTERN) PYSEQ_FRAME_PATTERN = os.getenv("PYSEQ_FRAME_PATTERN", DEFAULT_FRAME_PATTERN) +PYSEQ_ALLOW_NEGATIVE_FRAMES = os.getenv("PYSEQ_ALLOW_NEGATIVE_FRAMES", "0") + +# regex patterns for explicit frame-range syntax parsing/formatting +DEFAULT_FRAME_RANGE_SEGMENT_PATTERN = ( + r"^\s*(?P-?\d+)(?:\s*-\s*(?P-?\d+)(?:\s*x\s*(?P\d+))?)?\s*$" +) +DEFAULT_FRAME_RANGE_TEXT_PATTERN = ( + r"-?\d+(?:\s*-\s*-?\d+(?:\s*x\s*\d+)?)?(?:\s*,\s*-?\d+(?:\s*-\s*-?\d+(?:\s*x\s*\d+)?)?)*" +) +DEFAULT_SERIALIZED_RANGE_PATTERN = rf"\[[^\]]+\]|\s+(?:{DEFAULT_FRAME_RANGE_TEXT_PATTERN})\s*$" +DEFAULT_EMBEDDED_RANGE_PATTERN = rf"^(?P.+?)(?P\[(?:[^\]]+)\]|{DEFAULT_FRAME_RANGE_TEXT_PATTERN})(?P\.[^/\s]+)$" + + +def allow_negative_frames() -> bool: + """Return True when explicit negative frame syntax is enabled.""" + return os.getenv("PYSEQ_ALLOW_NEGATIVE_FRAMES", PYSEQ_ALLOW_NEGATIVE_FRAMES) == "1" + + +def get_effective_frame_pattern(pattern: str = DEFAULT_FRAME_PATTERN) -> str: + """Return the configured frame pattern, promoting the default when needed.""" + if allow_negative_frames() and pattern == DEFAULT_FRAME_PATTERN: + return DEFAULT_SIGNED_FRAME_PATTERN + return pattern def set_frame_pattern(pattern: str = DEFAULT_FRAME_PATTERN): @@ -65,13 +90,18 @@ def set_frame_pattern(pattern: str = DEFAULT_FRAME_PATTERN): :param pattern: The regex pattern to use for matching frame numbers. """ global frames_re + global digits_re global PYSEQ_FRAME_PATTERN PYSEQ_FRAME_PATTERN = pattern try: - frames_re = re.compile(pattern) + compiled = re.compile(get_effective_frame_pattern(pattern)) + frames_re = compiled + digits_re = compiled except Exception as e: print("Error: Invalid regex pattern: %s" % e) - frames_re = re.compile(DEFAULT_FRAME_PATTERN) + fallback = re.compile(DEFAULT_FRAME_PATTERN) + frames_re = fallback + digits_re = fallback # set the default frame pattern diff --git a/lib/pyseq/frange.py b/lib/pyseq/frange.py new file mode 100644 index 0000000..c348573 --- /dev/null +++ b/lib/pyseq/frange.py @@ -0,0 +1,155 @@ +#!/usr/bin/env python +# +# Copyright (c) 2011-2026, Ryan Galloway (ryan@rsgalloway.com) +# + +"""Frame range parsing and formatting helpers.""" + +import re +from typing import List, Optional, Tuple + +from pyseq import config + +frame_range_segment_re = re.compile(config.DEFAULT_FRAME_RANGE_SEGMENT_PATTERN) +frame_range_text_re = re.compile(config.DEFAULT_FRAME_RANGE_TEXT_PATTERN) +serialized_range_re = re.compile(config.DEFAULT_SERIALIZED_RANGE_PATTERN) +embedded_range_re = re.compile(config.DEFAULT_EMBEDDED_RANGE_PATTERN) + + +def has_serialized_range(text: str) -> bool: + """Return True when text includes explicit serialized frame range syntax.""" + return bool(serialized_range_re.search(text)) + + +def split_embedded_frame_range(text: str) -> Optional[Tuple[str, str, str]]: + """Split `headtail` strings like `plate.2-4.exr` into components.""" + match = embedded_range_re.match(text) + if not match: + return None + return match.group("head"), match.group("range"), match.group("tail") + + +def parse_frame_range(range_text: str) -> List[int]: + """Parse an extended frame range string into a list of frame numbers.""" + text = range_text.strip() + if not text: + return [] + if text.startswith("[") and text.endswith("]"): + text = text[1:-1].strip() + if not text: + return [] + + frames = [] + for segment in text.split(","): + segment = segment.strip() + if not segment: + raise ValueError(f"Invalid frame range syntax: {range_text}") + match = frame_range_segment_re.match(segment) + if not match: + raise ValueError(f"Invalid frame range syntax: {segment}") + + start = int(match.group("start")) + end = match.group("end") + step = match.group("step") + + if start < 0 and not config.allow_negative_frames(): + raise ValueError("Negative frame ranges require PYSEQ_ALLOW_NEGATIVE_FRAMES=1") + + if end is None: + frames.append(start) + continue + + end = int(end) + if end < 0 and not config.allow_negative_frames(): + raise ValueError("Negative frame ranges require PYSEQ_ALLOW_NEGATIVE_FRAMES=1") + step = int(step) if step is not None else 1 + if step <= 0: + raise ValueError(f"Frame step must be positive: {segment}") + + direction = 1 if end >= start else -1 + stop = end + direction + frames.extend(range(start, stop, direction * step)) + + return frames + + +def _frame_groups(frames: List[int]): + """Return canonical arithmetic groups for a sorted frame list.""" + unique_frames = sorted(set(frames)) + if not unique_frames: + return [] + if len(unique_frames) == 1: + return [(unique_frames[0], unique_frames[0], None, 1)] + + groups = [] + start = unique_frames[0] + prev = unique_frames[0] + step = None + count = 1 + + for frame in unique_frames[1:]: + diff = frame - prev + if step is None: + step = diff + prev = frame + count += 1 + continue + if diff == step: + prev = frame + count += 1 + continue + groups.append((start, prev, step, count)) + start = prev = frame + step = None + count = 1 + + groups.append((start, prev, step, count)) + return groups + + +def format_frame_range_explicit(frames: List[int], pad_with_brackets: bool = True) -> str: + """Format frames as explicit contiguous segments.""" + if not frames: + return "" + + frange = [] + start = end = None + for frame in sorted(set(frames)): + if start is None: + start = end = frame + continue + if frame != end + 1: + if start == end: + frange.append(str(start)) + else: + frange.append(f"{start}-{end}") + start = end = frame + continue + end = frame + + if start is not None: + if start == end: + frange.append(str(start)) + else: + frange.append(f"{start}-{end}") + + body = config.range_join.join(frange) + return f"[{body}]" if pad_with_brackets else body + + +def format_frame_range_stepped(frames: List[int]) -> str: + """Format frames using canonical stepped-range syntax.""" + if not frames: + return "" + + parts = [] + for start, end, step, count in _frame_groups(frames): + if count == 1: + parts.append(str(start)) + elif step == 1: + parts.append(f"{start}-{end}") + elif count == 2: + parts.extend([str(start), str(end)]) + else: + parts.append(f"{start}-{end}x{step}") + return ",".join(parts) diff --git a/lib/pyseq/lss.py b/lib/pyseq/lss.py index 89a8333..abd811a 100755 --- a/lib/pyseq/lss.py +++ b/lib/pyseq/lss.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Copyright (c) 2011-2025, Ryan Galloway (ryan@rsgalloway.com) +# Copyright (c) 2011-2026, Ryan Galloway (ryan@rsgalloway.com) # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: @@ -41,8 +41,8 @@ from pyseq import __version__, get_sequences from pyseq import seq as pyseq -from pyseq.util import cli_catch_keyboard_interrupt from pyseq import walk +from pyseq.cli import cli_catch_keyboard_interrupt def tree(source: str, level: Optional[int], seq_format: str): diff --git a/lib/pyseq/scopy.py b/lib/pyseq/scopy.py index 4e3a8bd..da74c22 100644 --- a/lib/pyseq/scopy.py +++ b/lib/pyseq/scopy.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Copyright (c) 2011-2025, Ryan Galloway (ryan@rsgalloway.com) +# Copyright (c) 2011-2026, Ryan Galloway (ryan@rsgalloway.com) # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: @@ -33,18 +33,15 @@ Contains the main scopy functions for the pyseq module. """ -import sys -import os import argparse +import os import shutil +import sys from typing import Optional import pyseq -from pyseq.util import ( - cli_catch_keyboard_interrupt, - parse_destination_reference, - resolve_sequence_reference, -) +from pyseq.cli import cli_catch_keyboard_interrupt +from pyseq.util import parse_destination_reference, resolve_sequence_reference def copy_sequence( @@ -149,17 +146,11 @@ def main(): dest_spec = parse_destination_reference(dest, seq) if len(sources) > 1 and dest_spec["kind"] != "directory": - raise ValueError( - "destination must be a directory when copying multiple sources" - ) + raise ValueError("destination must be a directory when copying multiple sources") rename = dest_spec["rename"] pad = args.pad if dest_spec["kind"] == "directory" else dest_spec["pad"] - renumber = ( - args.renumber - if dest_spec["kind"] == "directory" - else dest_spec["renumber"] - ) + renumber = args.renumber if dest_spec["kind"] == "directory" else dest_spec["renumber"] copy_sequence( seq, diff --git a/lib/pyseq/sdiff.py b/lib/pyseq/sdiff.py index db534bd..e5688ce 100644 --- a/lib/pyseq/sdiff.py +++ b/lib/pyseq/sdiff.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Copyright (c) 2011-2025, Ryan Galloway (ryan@rsgalloway.com) +# Copyright (c) 2011-2026, Ryan Galloway (ryan@rsgalloway.com) # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: @@ -38,7 +38,7 @@ import sys import pyseq -from pyseq.util import cli_catch_keyboard_interrupt +from pyseq.cli import cli_catch_keyboard_interrupt from pyseq.util import resolve_sequence @@ -58,7 +58,7 @@ def diff_sequences( def intval(val): try: return int(val) - except: + except (TypeError, ValueError): return None diff = { diff --git a/lib/pyseq/seq.py b/lib/pyseq/seq.py index 2bfc17d..3f4f23c 100755 --- a/lib/pyseq/seq.py +++ b/lib/pyseq/seq.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Copyright (c) 2011-2025, Ryan Galloway (ryan@rsgalloway.com) +# Copyright (c) 2011-2026, Ryan Galloway (ryan@rsgalloway.com) # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: @@ -30,27 +30,18 @@ # ----------------------------------------------------------------------------- __doc__ = """ -Contains the main pyseq classes and functions. +Core sequence types and parsing/formatting helpers. """ import functools import os import re -import traceback -import warnings from collections import deque from glob import glob, iglob -from typing import List, Callable, Union +from typing import Callable, List, Union from pyseq import config -from pyseq.util import _ext_key -from pyseq.config import ( - default_format, - format_re, - global_format, - range_join, - strict_pad, -) +from pyseq.config import default_format, format_re, global_format, range_join, strict_pad class SequenceError(Exception): @@ -65,28 +56,71 @@ class FormatError(Exception): pass +def _frame_width(frame: str) -> int: + """Return the digit width of a frame token, excluding any sign.""" + return len(frame[1:] if frame.startswith("-") else frame) + + +def _natural_key(x: str): + """Split a string into characters and digits for natural sorting.""" + return [int(c) if c.isdigit() else c.lower() for c in re.split(r"(\d+)", x)] + + +def _ext_key(x: str): + """Sort by extension first, then natural order of the basename.""" + name, ext = os.path.splitext(x) + return [ext] + _natural_key(name) + + +# Import frame-range helpers lazily so default pyseq/lss imports do not pay +# frange regex compilation costs unless range parsing/formatting is used. +def _format_frame_range_explicit(frames: List[int], pad_with_brackets: bool = True) -> str: + from pyseq.frange import format_frame_range_explicit + + return format_frame_range_explicit(frames, pad_with_brackets=pad_with_brackets) + + +def _format_frame_range_stepped(frames: List[int]) -> str: + from pyseq.frange import format_frame_range_stepped + + return format_frame_range_stepped(frames) + + +def _parse_frame_range(range_text: str) -> List[int]: + from pyseq.frange import parse_frame_range + + return parse_frame_range(range_text) + + def padsize(item, frame): """ - Determines the pad size for a given Item. Return value may depend on - whether strict padding is enabled or not. + Determine the pad size for a given Item. + + The return value may depend on whether strict padding is enabled. For example: the file item.001.exr will have a pad size of 3, and the file test.001001.exr will have a pad size of 6. :param item: Item object. - :param frame: the frame number as a string. - :returns: the size of the frame pad as an int. + Signed frames use the digit width only; the leading ``-`` does not + contribute to the padding width. + + :param frame: The frame number token as a string. + :returns: The size of the frame pad as an int. """ # strict: frame size (%d) must match between frames (default) # for example: test.09.jpg, test.10.jpg, test.11.jpg + width = _frame_width(frame) + digits = frame[1:] if frame.startswith("-") else frame + if strict_pad: - return item.pad or len(frame) + return item.pad or width # not strict: frame size can change between frames # for example: test.9.jpg, test.10.jpg, test.11.jpg else: - return item.pad or len(frame) if frame.startswith("0") else 0 + return item.pad or width if digits.startswith("0") else 0 class Item(str): @@ -102,17 +136,18 @@ def __init__(self, item: Union[str, os.PathLike]): """ super(Item, self).__init__() self.item = item - self.__path = getattr(item, "path", None) - if self.__path is None: + if isinstance(item, Item): + self.__path = item.path + else: self.__path = str(item) self.__filename = os.path.basename(self.__path) self.__number_matches = [] - self.__parts = config.frames_re.split(self.name) + self.__parts = config.frames_re.split(self.__filename) self.__stat = None # modified by self.is_sibling() self.frame = None - self.head = self.name + self.head = self.__filename self.tail = "" self.pad = None @@ -332,6 +367,11 @@ def is_sibling(self, item: str): return is_sibling +def _ensure_item(item: Union[Item, str, os.PathLike]) -> Item: + """Return the original Item or wrap path-like input in an Item.""" + return item if isinstance(item, Item) else Item(item) + + class Sequence(list): """Extends list class with methods that handle item sequentialness. @@ -362,13 +402,13 @@ def __init__(self, items: List[str]): """ # otherwise Sequence consumes the list items = deque(items[::]) - super(Sequence, self).__init__([Item(items.popleft())]) + super(Sequence, self).__init__([_ensure_item(items.popleft())]) self.__missing = [] self.__dirty = False self.__frames = None while items: - f = Item(items.popleft()) + f = _ensure_item(items.popleft()) try: self.append(f) except SequenceError: @@ -385,13 +425,14 @@ def __attrs__(self): "e": self.end, "f": self.frames, "m": self.missing, - "M": functools.partial(self._get_framerange, self.missing(), missing=True), + "M": lambda: self._get_framerange(self.missing(), missing=True), "d": lambda *x: self.size, "H": lambda *x: self.human, "D": self.directory, "p": self._get_padding, - "r": functools.partial(self._get_framerange, self.frames(), missing=False), - "R": functools.partial(self._get_framerange, self.frames(), missing=True), + "r": lambda: self._get_framerange(self.frames(), missing=False), + "R": lambda: self._get_framerange(self.frames(), missing=True), + "x": lambda: _format_frame_range_stepped(self.frames()), "h": self.head, "t": self.tail, } @@ -514,6 +555,8 @@ def format(self, fmt: str = global_format): +-----------+--------------------------------------+ | ``%R`` | explicit broken range, [1-10, 15-20] | +-----------+--------------------------------------+ + | ``%x`` | stepped explicit range, 1-10x2, 20 | + +-----------+--------------------------------------+ | ``%d`` | disk usage | +-----------+--------------------------------------+ | ``%H`` | disk usage (human readable) | @@ -539,6 +582,7 @@ def format(self, fmt: str = global_format): "p": "s", "r": "s", "R": "s", + "x": "s", "d": "s", "H": "s", "D": "s", @@ -698,9 +742,7 @@ def includes(self, item: Union[str, Item]): for anchor in anchors: if anchor.is_sibling(item): - return item.name.startswith(canonical_head) and item.name.endswith( - canonical_tail - ) + return item.name.startswith(canonical_head) and item.name.endswith(canonical_tail) return False @@ -799,8 +841,7 @@ def reIndex(self, offset: int, padding: int = None): if offset > 0: gen = ( - (image, frame) - for (image, frame) in zip(reversed(self), reversed(self.frames())) + (image, frame) for (image, frame) in zip(reversed(self), reversed(self.frames())) ) else: gen = ((image, frame) for (image, frame) in zip(self, self.frames())) @@ -816,6 +857,9 @@ def reIndex(self, offset: int, padding: int = None): shutil.move(oldName, newName) except Exception as err: + import traceback + import warnings + warnings.warn( "%s during reIndex %s -> %s: \n%s" % ( @@ -852,9 +896,6 @@ def _get_framerange(self, frames: List[int], missing: bool = True): :return: Formatted frame range string. """ - frange = [] - start = "" - end = "" if not missing: if frames: return "%s-%s" % (self.start(), self.end()) @@ -864,28 +905,22 @@ def _get_framerange(self, frames: List[int], missing: bool = True): if not frames: return "" - for i in range(0, len(frames)): - frame = frames[i] + frange = [] + expanded = [] + saw_ranges = False + for frame in frames: if isinstance(frame, range): + saw_ranges = True if frame.start != frame.stop: frange.append("%s-%s" % (frame.start, frame.stop - 1)) continue - prev = frames[i - 1] - if i != 0 and frame != prev + 1: - if start != end: - frange.append("%s-%s" % (str(start), str(end))) - elif start == end: - frange.append(str(start)) - start = end = frame - continue - if start == "" or int(start) > frame: - start = frame - if end == "" or int(end) < frame: - end = frame - if start == end: - frange.append(str(start)) - else: - frange.append("%s-%s" % (str(start), str(end))) + expanded.append(frame) + + explicit = _format_frame_range_explicit(expanded, pad_with_brackets=False) + if explicit: + frange.extend(explicit.split(range_join)) + if saw_ranges: + frange.append("") return "[%s]" % range_join.join(frange) def _get_frames(self): @@ -946,7 +981,7 @@ def diff(f1: Union[str, Item], f2: Union[str, Item]): if len(f1.number_matches) == len(f2.number_matches): for m1, m2 in zip(f1.number_matches, f2.number_matches): if (m1.start() == m2.start()) and (m1.group() != m2.group()): - if strict_pad is True and (len(m1.group()) != len(m2.group())): + if strict_pad is True and (_frame_width(m1.group()) != _frame_width(m2.group())): continue d.append( { @@ -960,7 +995,7 @@ def diff(f1: Union[str, Item], f2: Union[str, Item]): def uncompress(seq_string: str, fmt: str = global_format): - """Basic uncompression or deserialization of a compressed sequence string. + """Deserialize a compressed sequence string into a Sequence. For example: @@ -985,9 +1020,14 @@ def uncompress(seq_string: str, fmt: str = global_format): >>> len(seq) 100 + >>> seq = pyseq.uncompress('render.%04d.exr 1001-1010x3', fmt='%h%p%t %x') + >>> print(seq.frames()) + [1001, 1004, 1007, 1010] + :param seq_string: Compressed sequence string. :param fmt: Format of sequence string. - :return: :class:`.Sequence` instance. + :return: :class:`.Sequence` instance, or ``None`` when the string does not + match ``fmt``. """ dirname = os.path.dirname(seq_string) @@ -998,14 +1038,25 @@ def uncompress(seq_string: str, fmt: str = global_format): name = os.path.basename(seq_string) # map of directives to regex + allow_negative = config.allow_negative_frames() + range_pattern = ( + r"-?\d+\s*-\s*-?\d+(?:\s*x\s*\d+)?" if allow_negative else r"\d+\s*-\s*\d+(?:\s*x\s*\d+)?" + ) + explicit_pattern = ( + r"-?\d+(?:\s*-\s*-?\d+(?:\s*x\s*\d+)?)?(?:\s*,\s*-?\d+(?:\s*-\s*-?\d+(?:\s*x\s*\d+)?)?)*" + if allow_negative + else r"\d+(?:\s*-\s*\d+(?:\s*x\s*\d+)?)?(?:\s*,\s*\d+(?:\s*-\s*\d+(?:\s*x\s*\d+)?)?)*" + ) + remap = { "s": r"\d+", "e": r"\d+", "l": r"\d+", "h": r"(.+)?", "t": r"(\S+)?", - "r": r"\d+-\d+", - "R": r"\[[\d\s?\-%s?]+\]" % re.escape(range_join), + "r": range_pattern, + "R": r"\[[^\]]+\]", + "x": explicit_pattern, "p": r"%\d+d", "m": r"\[.*\]", "f": r"\[.*\]", @@ -1025,7 +1076,7 @@ def uncompress(seq_string: str, fmt: str = global_format): regex = re.compile(fmt) match = regex.match(name) - frames = [] + frame_values = [] missing = [] s = None e = None @@ -1033,6 +1084,8 @@ def uncompress(seq_string: str, fmt: str = global_format): if not match: return + from ast import literal_eval + try: pad = match.group("p") @@ -1041,71 +1094,78 @@ def uncompress(seq_string: str, fmt: str = global_format): try: R = match.group("R") - R = R[1:-1] - number_groups = R.split(range_join) - pad_len = 0 - for number_group in number_groups: - if "-" in number_group: - splits = number_group.split("-") - pad_len = max(pad_len, len(splits[0]), len(splits[1])) - start = int(splits[0]) - end = int(splits[1]) - frames.extend(range(start, end + 1)) - - else: - end = int(number_group) - pad_len = max(pad_len, len(number_group)) - frames.append(end) + frame_values = _parse_frame_range(R) + pad_len = max((len(str(abs(frame))) for frame in frame_values), default=0) if pad == "%d" and pad_len != 0: pad = "%0" + str(pad_len) + "d" except IndexError: try: r = match.group("r") - s, e = r.split("-") - frames = range(int(s), int(e) + 1) + frame_values = _parse_frame_range(r) + if frame_values: + s = frame_values[0] + e = frame_values[-1] except IndexError: - s = match.group("s") - e = match.group("e") + try: + x = match.group("x") + frame_values = _parse_frame_range(x) + except IndexError: + s = match.group("s") + e = match.group("e") + if s is not None and e is not None: + frame_values = _parse_frame_range(f"{s}-{e}") try: - frames = eval(match.group("f")) + frame_values = literal_eval(match.group("f")) except IndexError: pass try: - missing = eval(match.group("m")) + missing = literal_eval(match.group("m")) except IndexError: pass items = [] + emitted_frames = [] + head = match.groupdict().get("h", "") + tail = match.groupdict().get("t", "") + item_pad = 0 if pad == "%d" else int(re.search(r"\d+", pad).group()) if missing: - for i in range(int(s), int(e) + 1): + for i in _parse_frame_range(f"{s}-{e}"): if i in missing: continue - f = pad % i - name = "%s%s%s" % ( - match.groupdict().get("h", ""), - f, - match.groupdict().get("t", ""), - ) - items.append(Item(os.path.join(dirname, name))) + f = ("-" if i < 0 else "") + (pad % abs(i)) + name = "%s%s%s" % (head, f, tail) + item = Item(os.path.join(dirname, name)) + item.frame = i + item.head = head + item.tail = tail + item.pad = item_pad + items.append(item) + emitted_frames.append(i) else: - for i in frames: - f = pad % i - name = "%s%s%s" % ( - match.groupdict().get("h", ""), - f, - match.groupdict().get("t", ""), - ) - items.append(Item(os.path.join(dirname, name))) + for i in frame_values: + f = ("-" if i < 0 else "") + (pad % abs(i)) + name = "%s%s%s" % (head, f, tail) + item = Item(os.path.join(dirname, name)) + item.frame = i + item.head = head + item.tail = tail + item.pad = item_pad + items.append(item) + emitted_frames.append(i) seqs = get_sequences(items) if seqs: + if emitted_frames: + seq = seqs[0] + seq._Sequence__frames = sorted(emitted_frames) + seq._Sequence__missing = None return seqs[0] return seqs @@ -1157,6 +1217,9 @@ def get_sequences(source: str, frame_pattern: str = config.PYSEQ_FRAME_PATTERN): if isinstance(source, list): items = sorted(source, key=lambda x: str(x)) + if items and isinstance(items[0], Item): + for item in items: + item._Item__parts = config.frames_re.split(item.name) elif isinstance(source, str): if os.path.isdir(source): @@ -1171,7 +1234,7 @@ def get_sequences(source: str, frame_pattern: str = config.PYSEQ_FRAME_PATTERN): # organize the items into sequences while items: - item = Item(items.popleft()) + item = _ensure_item(items.popleft()) found = False for seq in reversed(seqs): if seq.includes(item): diff --git a/lib/pyseq/sfind.py b/lib/pyseq/sfind.py index 655445c..1343598 100644 --- a/lib/pyseq/sfind.py +++ b/lib/pyseq/sfind.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Copyright (c) 2011-2025, Ryan Galloway (ryan@rsgalloway.com) +# Copyright (c) 2011-2026, Ryan Galloway (ryan@rsgalloway.com) # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: @@ -39,7 +39,7 @@ import sys import pyseq -from pyseq.util import cli_catch_keyboard_interrupt +from pyseq.cli import cli_catch_keyboard_interrupt def walk_and_collect_sequences( @@ -87,9 +87,7 @@ def main(): if not os.path.isdir(path): print(f"sfind: {path} is not a directory", file=sys.stderr) continue - for seq in walk_and_collect_sequences( - path, include_hidden=args.all, pattern=args.name - ): + for seq in walk_and_collect_sequences(path, include_hidden=args.all, pattern=args.name): print(seq) return 0 diff --git a/lib/pyseq/smove.py b/lib/pyseq/smove.py index cdbe79a..be2a1ef 100644 --- a/lib/pyseq/smove.py +++ b/lib/pyseq/smove.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Copyright (c) 2011-2025, Ryan Galloway (ryan@rsgalloway.com) +# Copyright (c) 2011-2026, Ryan Galloway (ryan@rsgalloway.com) # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: @@ -33,18 +33,15 @@ Contains the main smove functions for the pyseq module. """ -import sys -import os import argparse +import os import shutil +import sys from typing import Optional import pyseq -from pyseq.util import ( - cli_catch_keyboard_interrupt, - parse_destination_reference, - resolve_sequence_reference, -) +from pyseq.cli import cli_catch_keyboard_interrupt +from pyseq.util import parse_destination_reference, resolve_sequence_reference def move_sequence( @@ -149,17 +146,11 @@ def main(): dest_spec = parse_destination_reference(dest, seq) if len(sources) > 1 and dest_spec["kind"] != "directory": - raise ValueError( - "destination must be a directory when moving multiple sources" - ) + raise ValueError("destination must be a directory when moving multiple sources") rename = dest_spec["rename"] pad = args.pad if dest_spec["kind"] == "directory" else dest_spec["pad"] - renumber = ( - args.renumber - if dest_spec["kind"] == "directory" - else dest_spec["renumber"] - ) + renumber = args.renumber if dest_spec["kind"] == "directory" else dest_spec["renumber"] dest_dir = dest_spec["dest_dir"] move_sequence( diff --git a/lib/pyseq/sremove.py b/lib/pyseq/sremove.py index 20fe685..7140856 100644 --- a/lib/pyseq/sremove.py +++ b/lib/pyseq/sremove.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Copyright (c) 2011-2025, Ryan Galloway (ryan@rsgalloway.com) +# Copyright (c) 2011-2026, Ryan Galloway (ryan@rsgalloway.com) # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: @@ -38,7 +38,8 @@ import sys import pyseq -from pyseq.util import cli_catch_keyboard_interrupt, resolve_sequence_reference +from pyseq.cli import cli_catch_keyboard_interrupt +from pyseq.util import resolve_sequence_reference def remove_sequence( diff --git a/lib/pyseq/sstat.py b/lib/pyseq/sstat.py index f25540b..02f0486 100644 --- a/lib/pyseq/sstat.py +++ b/lib/pyseq/sstat.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Copyright (c) 2011-2025, Ryan Galloway (ryan@rsgalloway.com) +# Copyright (c) 2011-2026, Ryan Galloway (ryan@rsgalloway.com) # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: @@ -40,7 +40,7 @@ import sys import pyseq -from pyseq.util import cli_catch_keyboard_interrupt +from pyseq.cli import cli_catch_keyboard_interrupt from pyseq.util import resolve_sequence @@ -79,9 +79,7 @@ def format_time_range(t1, t2): return f"{format_time(t1)}.. {format_time(t2)}" print(f"Sequence: {str(seq)}") - print( - f"Size: {seq.format('%H'):>8} Frames: {seq.format('%l'):>5} Padding: {seq.pad}" - ) + print(f"Size: {seq.format('%H'):>8} Frames: {seq.format('%l'):>5} Padding: {seq.pad}") missing = seq.format("%M") print(f"Missing: {missing if missing else 'none'}") print(f"Head: {seq.head()}") diff --git a/lib/pyseq/stree.py b/lib/pyseq/stree.py index da81506..8bba376 100644 --- a/lib/pyseq/stree.py +++ b/lib/pyseq/stree.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Copyright (c) 2011-2025, Ryan Galloway (ryan@rsgalloway.com) +# Copyright (c) 2011-2026, Ryan Galloway (ryan@rsgalloway.com) # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: @@ -39,7 +39,7 @@ import pyseq from pyseq import config -from pyseq.util import cli_catch_keyboard_interrupt +from pyseq.cli import cli_catch_keyboard_interrupt def get_tree_tokens(): diff --git a/lib/pyseq/util.py b/lib/pyseq/util.py index 442d317..235a90d 100644 --- a/lib/pyseq/util.py +++ b/lib/pyseq/util.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Copyright (c) 2011-2025, Ryan Galloway (ryan@rsgalloway.com) +# Copyright (c) 2011-2026, Ryan Galloway (ryan@rsgalloway.com) # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: @@ -29,17 +29,16 @@ # POSSIBILITY OF SUCH DAMAGE. # ----------------------------------------------------------------------------- -import functools import fnmatch +import functools import glob import os import re -import sys import warnings from typing import Optional import pyseq -from pyseq.config import range_join +from pyseq import config def deprecated(func): @@ -59,20 +58,6 @@ def inner(*args, **kwargs): return inner -def cli_catch_keyboard_interrupt(func): - """Return exit code 1 instead of a traceback on Ctrl-C.""" - - @functools.wraps(func) - def inner(*args, **kwargs): - try: - return func(*args, **kwargs) - except KeyboardInterrupt: - print("stopping...", file=sys.stderr) - return 1 - - return inner - - def _natural_key(x: str): """Splits a string into characters and digits. @@ -82,29 +67,6 @@ def _natural_key(x: str): return [int(c) if c.isdigit() else c.lower() for c in re.split(r"(\d+)", x)] -def _ext_key(x: str): - """Similar to `_natural_key` except this one uses the file extension at - the head of split string. This fixes issues with files that are named - similar but with different file extensions: - - This example: - - file.001.jpg - file.001.tiff - file.002.jpg - file.002.tiff - - Would get properly sorted into: - - file.001.jpg - file.002.jpg - file.001.tiff - file.002.tiff - """ - name, ext = os.path.splitext(x) - return [ext] + _natural_key(name) - - def is_compressed_format_string(s: str) -> bool: """Check if the string is a compressed format string. A compressed format string is a string that contains a format specifier for integers, such as @@ -127,12 +89,13 @@ def natural_sort(items: list): return sorted(items, key=_natural_key) -def resolve_sequence(sequence_string: str): +def resolve_sequence(sequence_string: str, include_negative: bool = False): """Given a compressed sequence string like 'file.%04d.png' or '/path/to/file.%04d.png', return a `Sequence` object of matching files on disk. :param sequence_string: The compressed sequence string to be uncompressed. + :param include_negative: When True, also scan for signed padded frames. :return: A pyseq.Sequence object of matching files. """ @@ -148,17 +111,27 @@ def resolve_sequence(sequence_string: str): padding = match.group(1) if padding: pad = int(padding) - glob_part = filename.replace(f"%0{pad}d", "?" * pad) - regex_pattern = re.escape(filename).replace( - f"%0{pad}d", r"\d{" + str(pad) + r"}" - ) + frame_token = f"%0{pad}d" + positive_glob = filename.replace(frame_token, "?" * pad) + negative_glob = filename.replace(frame_token, "-" + ("?" * pad)) + frame_pattern = r"-?\d{" + str(pad) + r"}" if include_negative else r"\d{" + str(pad) + r"}" + regex_pattern = re.escape(filename).replace(frame_token, frame_pattern) else: - glob_part = filename.replace("%d", "*") - regex_pattern = re.escape(filename).replace("%d", r"\d+") - - # glob all files in the directory using glob pattern - glob_pattern = os.path.join(directory, glob_part) - candidate_files = glob.glob(glob_pattern) + positive_glob = filename.replace("%d", "*") + negative_glob = None + frame_pattern = r"-?\d+" if include_negative else r"\d+" + regex_pattern = re.escape(filename).replace("%d", frame_pattern) + + candidate_files = glob.glob(os.path.join(directory, positive_glob)) + if include_negative and negative_glob and negative_glob != positive_glob: + negative_files = glob.glob(os.path.join(directory, negative_glob)) + if candidate_files: + existing = set(candidate_files) + candidate_files.extend( + candidate for candidate in negative_files if candidate not in existing + ) + else: + candidate_files = negative_files # filter using regex (because glob pattern is wide) regex = re.compile(f"^{regex_pattern}$") @@ -196,64 +169,79 @@ def subset_sequence(seq, frames): sequences = pyseq.get_sequences(items) if not sequences: raise ValueError("No valid sequence found for requested frame subset") - return sequences[0] + + subset = sequences[0] + subset._Sequence__frames = sorted(frame_set) + subset._Sequence__missing = None + + # Preserve source sequence formatting metadata for resolved subsets. + for item in subset: + item.head = seq.head() + item.tail = seq.tail() + item.pad = seq.pad + + return subset def parse_explicit_sequence_string(reference: str): """Parse a serialized sequence string, including embedded range syntax.""" + from pyseq.frange import has_serialized_range, parse_frame_range, split_embedded_frame_range + dirname = os.path.dirname(reference) or "." basename = os.path.basename(reference) + has_explicit_range = has_serialized_range(basename) - embedded = re.match( - r"^(?P.+?)(?P\[(?:[^\]]+)\]|\d+-\d+)(?P\.[^/\s]+)$", - basename, + # Plain compressed sequence patterns like "file.%04d.exr" should be + # resolved against disk, not reinterpreted as explicit serialized ranges. + if is_compressed_format_string(basename) and not has_explicit_range: + return None + + embedded = ( + None if is_compressed_format_string(basename) else split_embedded_frame_range(basename) ) if embedded: - range_text = embedded.group("range") - frames = [] - if range_text.startswith("["): - for number_group in range_text[1:-1].split(range_join): - number_group = number_group.strip() - if not number_group: - continue - if "-" in number_group: - start, end = number_group.split("-", 1) - frames.extend(range(int(start), int(end) + 1)) - else: - frames.append(int(number_group)) - else: - start, end = range_text.split("-", 1) - frames = list(range(int(start), int(end) + 1)) + head, range_text, tail = embedded + frames = parse_frame_range(range_text) - items = [ - pyseq.Item( + items = [] + for frame in frames: + item = pyseq.Item( os.path.join( dirname, - f"{embedded.group('head')}{frame}{embedded.group('tail')}", + f"{head}{frame}{tail}", ) ) - for frame in frames - ] + item.frame = frame + item.head = head + item.tail = tail + item.pad = 0 + items.append(item) sequences = pyseq.get_sequences(items) if sequences: + seq = sequences[0] + seq._Sequence__frames = sorted(frames) + seq._Sequence__missing = None return { - "seq": sequences[0], + "seq": seq, "has_pad": False, } - formats = ( - ("%h%p%t %R", True), - ("%h%p%t %r", True), - ("%h%R%t", False), - ("%h%r%t", False), - ) - for fmt, has_pad in formats: - seq = pyseq.uncompress(reference, fmt=fmt) - if seq: - return { - "seq": seq, - "has_pad": has_pad, - } + if has_explicit_range: + formats = ( + ("%h%p%t %x", True), + ("%h%p%t %R", True), + ("%h%p%t %r", True), + ("%h%x%t", False), + ("%h%R%t", False), + ("%h%r%t", False), + ) + for fmt, has_pad in formats: + seq = pyseq.uncompress(reference, fmt=fmt) + if seq: + return { + "seq": seq, + "has_pad": has_pad, + } return None @@ -273,26 +261,27 @@ def resolve_sequence_reference(reference: str): requested_seq.tail(), ), ) - full_seq = resolve_sequence(pattern) + full_seq = resolve_sequence( + pattern, + include_negative=( + config.allow_negative_frames() + and any(frame < 0 for frame in requested_seq.frames()) + ), + ) else: sequences = pyseq.get_sequences(os.listdir(dirname)) candidates = [ seq for seq in sequences - if seq.head() == requested_seq.head() - and seq.tail() == requested_seq.tail() + if seq.head() == requested_seq.head() and seq.tail() == requested_seq.tail() ] candidates = [ - seq - for seq in candidates - if set(requested_seq.frames()).issubset(set(seq.frames())) + seq for seq in candidates if set(requested_seq.frames()).issubset(set(seq.frames())) ] if not candidates: raise FileNotFoundError(f"No sequence found matching {reference}") if len(candidates) > 1: - raise ValueError( - f"Multiple sequences found matching {reference}: {candidates}" - ) + raise ValueError(f"Multiple sequences found matching {reference}: {candidates}") full_seq = candidates[0] return subset_sequence(full_seq, requested_seq.frames()), dirname @@ -322,9 +311,7 @@ def parse_destination_reference(destination: str, source_seq): expected_frames = list(range(dest_frames[0], dest_frames[0] + len(source_seq))) if dest_seq.tail() != source_seq.tail(): - raise ValueError( - "Destination sequence pattern must preserve the source extension" - ) + raise ValueError("Destination sequence pattern must preserve the source extension") if dest_frames != expected_frames: raise ValueError("Destination explicit range must be contiguous") if len(dest_seq) != len(source_seq): @@ -354,9 +341,7 @@ def parse_destination_reference(destination: str, source_seq): tail = filename[match.end() :] if tail != source_seq.tail(): - raise ValueError( - "Destination sequence pattern must preserve the source extension" - ) + raise ValueError("Destination sequence pattern must preserve the source extension") return { "kind": "sequence", diff --git a/pyproject.toml b/pyproject.toml index 17edf1c..753cf45 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,7 +40,16 @@ classifiers = [ authors = [ { name = "Ryan Galloway", email = "ryan@rsgalloway.com" }, ] -optional-dependencies = { dev = ["pytest"], test = ["pytest"] } +optional-dependencies = { dev = ["pytest", "flake8==7.1.1", "mccabe==0.7.0", "isort==5.13.2", "black==24.8.0"], test = ["pytest"] } + +[tool.isort] +profile = "black" +line_length = 100 +skip_gitignore = true + +[tool.black] +line-length = 100 +target-version = ["py38"] [project.scripts] lss = "pyseq.lss:main" diff --git a/pyseq.env b/pyseq.env index 6a592ae..d580408 100755 --- a/pyseq.env +++ b/pyseq.env @@ -13,6 +13,9 @@ all: &all # frame numbers start with an underscore, e.g. file_v1_1001.exr # PYSEQ_FRAME_PATTERN: _\d+ + # allow signed frame numbers in explicit ranges and filename discovery + # PYSEQ_ALLOW_NEGATIVE_FRAMES: 1 + # sequence string format: 4 file01_%04d.exr [40-43] (default) PYSEQ_GLOBAL_FORMAT: "%4l %h%p%t %R" diff --git a/scripts/benchmark.py b/scripts/benchmark.py new file mode 100644 index 0000000..bdef478 --- /dev/null +++ b/scripts/benchmark.py @@ -0,0 +1,724 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2011-2025, Ryan Galloway (ryan@rsgalloway.com) +# + +"""Simple local benchmark runner for pyseq hotspots.""" + +import argparse +import cProfile +import io +import json +import os +import platform +import pstats +import statistics +import subprocess +import sys +import tempfile +import time +from contextlib import contextmanager +from datetime import datetime, timezone +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent +LIB_ROOT = REPO_ROOT / "lib" +SCRIPT_PATH = Path(__file__).resolve() +if str(LIB_ROOT) not in sys.path: + sys.path.insert(0, str(LIB_ROOT)) + +import pyseq # noqa: E402 +import pyseq.seq as pyseq_seq # noqa: E402 +from pyseq import get_sequences # noqa: E402 +from pyseq.util import resolve_sequence # noqa: E402 + +DEFAULT_SIZES = [100, 1000, 10000] +FULL_SIZES = [100, 1000, 10000, 50000] +DEFAULT_ITERATIONS = 5 +FULL_ITERATIONS = 7 +EXTRA_FILES_FACTOR = 0.1 +PROFILE_TOP_FUNCTIONS = 25 +DEFAULT_SCENARIOS = ["contiguous", "mixed"] + + +def median(values): + return statistics.median(values) if values else 0.0 + + +def mean(values): + return statistics.mean(values) if values else 0.0 + + +def pct_delta(new_value, old_value): + if old_value == 0: + return 0.0 + return ((new_value / old_value) - 1.0) * 100.0 + + +def parse_sizes(value): + return [int(part.strip()) for part in value.split(",") if part.strip()] + + +def parse_csv_list(value): + return [part.strip() for part in value.split(",") if part.strip()] + + +def detect_git_branch(): + try: + result = subprocess.run( + ["git", "branch", "--show-current"], + cwd=REPO_ROOT, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + return result.stdout.strip() or "detached" + except Exception: + return "unknown" + + +def detect_git_sha(): + try: + result = subprocess.run( + ["git", "rev-parse", "--short", "HEAD"], + cwd=REPO_ROOT, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + return result.stdout.strip() + except Exception: + return "unknown" + + +def safe_path_token(value): + return value.replace(os.sep, "-").replace("/", "-") + + +def make_default_profiles_dir(branch, commit): + timestamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + return ( + REPO_ROOT + / "tmp" + / "benchmarks" + / "profiles" + / f"{timestamp}-{safe_path_token(branch)}-{safe_path_token(commit)}" + ) + + +def benchmark_call(func, iterations, warmups=1): + for _ in range(warmups): + func() + + samples = [] + for _ in range(iterations): + start = time.perf_counter() + func() + samples.append(time.perf_counter() - start) + + return { + "iterations": iterations, + "samples_s": samples, + "median_s": median(samples), + "mean_s": mean(samples), + "min_s": min(samples), + "max_s": max(samples), + } + + +def create_contiguous_dataset(root, size, extra_files_factor=EXTRA_FILES_FACTOR): + dataset = root / f"contiguous_{size}" + dataset.mkdir(parents=True, exist_ok=True) + + for frame in range(1, size + 1): + (dataset / f"renderA.{frame:08d}.exr").touch() + + extra_count = max(1, int(size * extra_files_factor)) + for index in range(extra_count): + (dataset / f"note_{index:08d}.txt").touch() + + return { + "path": dataset, + "sequence_pattern": str(dataset / "renderA.%08d.exr"), + "file_names": sorted(os.listdir(dataset)), + } + + +def _partition_count(total, parts): + if parts <= 0: + return [] + base, remainder = divmod(total, parts) + return [base + (1 if index < remainder else 0) for index in range(parts)] + + +def create_mixed_dataset(root, size, extra_files_factor=EXTRA_FILES_FACTOR): + dataset = root / f"mixed_{size}" + dataset.mkdir(parents=True, exist_ok=True) + + primary_count = max(1, int(size * 0.75)) + secondary_count = max(1, int(size * 0.08)) + tertiary_count = max(1, int(size * 0.04)) + quaternary_count = max(1, int(size * 0.04)) + + gap_stride = max(50, size // 12 or 1) + segment_lengths = _partition_count(primary_count, 9) + frame = 1 + for segment_length in segment_lengths: + for _ in range(segment_length): + (dataset / f"base.{frame:08d}.png").touch() + frame += 1 + frame += gap_stride + + secondary_start = 1001 + tertiary_start = 1001 + quaternary_start = 1001 + for offset in range(secondary_count): + (dataset / f"big_buck_bunny_1080p_h264.{secondary_start + offset:08d}.png").touch() + for offset in range(tertiary_count): + (dataset / f"big_buck_bunny_1080p_h264_test.{tertiary_start + offset:08d}.png").touch() + for offset in range(quaternary_count): + (dataset / f"test.{quaternary_start + offset:08d}.png").touch() + + extra_count = max(1, int(size * extra_files_factor)) + for index in range(extra_count): + (dataset / f"notes_{index:08d}.txt").touch() + + for tile in range(max(2, size // 2000 + 1)): + for frame in range(101, 111): + (dataset / f"bnc01_TinkSO_tx_{tile}_ty_{tile}.{frame:04d}.tif").touch() + + return { + "path": dataset, + "sequence_pattern": str(dataset / "base.%08d.png"), + "file_names": sorted(os.listdir(dataset)), + } + + +def create_existing_dataset(path, sequence_pattern=None, glob_pattern=None): + dataset = Path(path).resolve() + file_names = sorted(os.listdir(dataset)) + if glob_pattern: + import fnmatch + + file_names = [name for name in file_names if fnmatch.fnmatch(name, glob_pattern)] + + return { + "path": dataset, + "sequence_pattern": sequence_pattern, + "file_names": file_names, + } + + +def create_dataset(root, size, scenario): + if scenario == "contiguous": + return create_contiguous_dataset(root, size) + if scenario == "mixed": + return create_mixed_dataset(root, size) + raise ValueError(f"Unknown benchmark scenario: {scenario}") + + +def run_lss(path, glob_pattern=None): + env = os.environ.copy() + env["PYTHONPATH"] = ( + str(LIB_ROOT) + if not env.get("PYTHONPATH") + else str(LIB_ROOT) + os.pathsep + env["PYTHONPATH"] + ) + target = str(path) + if glob_pattern: + target = str(Path(path) / glob_pattern) + subprocess.run( + [sys.executable, "-m", "pyseq.lss", target], + cwd=REPO_ROOT, + env=env, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + + +def profile_python_callable(func, stem, profiles_dir): + profiler = cProfile.Profile() + profiler.enable() + func() + profiler.disable() + + pstats_path = profiles_dir / f"{stem}.pstats" + text_path = profiles_dir / f"{stem}.txt" + profiler.dump_stats(str(pstats_path)) + + stream = io.StringIO() + stats = pstats.Stats(profiler, stream=stream).sort_stats("cumulative") + stats.print_stats(PROFILE_TOP_FUNCTIONS) + text_path.write_text(stream.getvalue(), encoding="utf-8") + + +def profile_lss_subprocess(path, stem, profiles_dir, glob_pattern=None): + env = os.environ.copy() + env["PYTHONPATH"] = ( + str(LIB_ROOT) + if not env.get("PYTHONPATH") + else str(LIB_ROOT) + os.pathsep + env["PYTHONPATH"] + ) + pstats_path = profiles_dir / f"{stem}.pstats" + text_path = profiles_dir / f"{stem}.txt" + target = str(path) + if glob_pattern: + target = str(Path(path) / glob_pattern) + + subprocess.run( + [ + sys.executable, + "-m", + "cProfile", + "-o", + str(pstats_path), + "-m", + "pyseq.lss", + target, + ], + cwd=REPO_ROOT, + env=env, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + + stream = io.StringIO() + stats = pstats.Stats(str(pstats_path), stream=stream).sort_stats("cumulative") + stats.print_stats(PROFILE_TOP_FUNCTIONS) + text_path.write_text(stream.getvalue(), encoding="utf-8") + + +def _path_stem(path): + return safe_path_token(str(Path(path).resolve())) + + +@contextmanager +def build_benchmarks( + sizes=None, + scenarios=None, + path=None, + sequence_pattern=None, + glob_pattern=None, +): + with tempfile.TemporaryDirectory() as tmpdir: + cases = [] + fixtures = [] + + if path: + fixtures.append( + { + "label": _path_stem(path), + **create_existing_dataset( + path, + sequence_pattern=sequence_pattern, + glob_pattern=glob_pattern, + ), + } + ) + else: + root = Path(tmpdir) + selected_scenarios = scenarios or DEFAULT_SCENARIOS + for size in sizes: + for scenario in selected_scenarios: + fixture = create_dataset(root, size, scenario) + fixtures.append({"label": f"{scenario}_{size}", **fixture}) + + for fixture in fixtures: + dataset_path = fixture["path"] + file_names = fixture["file_names"] + sequence_pattern = fixture["sequence_pattern"] + label = fixture["label"] + + cases.extend( + [ + { + "name": f"get_sequences_list_{label}", + "callable": lambda file_names=file_names: get_sequences(file_names), + "profile_kind": "python", + }, + { + "name": f"get_sequences_dir_{label}", + "callable": lambda dataset_path=dataset_path, glob_pattern=glob_pattern: get_sequences( + str(Path(dataset_path) / glob_pattern) + if glob_pattern + else str(dataset_path) + ), + "profile_kind": "python", + }, + { + "name": f"lss_{label}", + "callable": lambda dataset_path=dataset_path, glob_pattern=glob_pattern: run_lss( + dataset_path, glob_pattern=glob_pattern + ), + "profile_kind": "lss", + "path": dataset_path, + "glob_pattern": glob_pattern, + }, + ] + ) + if sequence_pattern: + cases.append( + { + "name": f"resolve_sequence_{label}", + "callable": lambda sequence_pattern=sequence_pattern: resolve_sequence( + sequence_pattern + ), + "profile_kind": "python", + } + ) + + yield cases + + +def run_benchmarks( + sizes, + scenarios, + iterations, + enable_profile, + profiles_dir, + path=None, + sequence_pattern=None, + glob_pattern=None, +): + with build_benchmarks( + sizes=sizes, + scenarios=scenarios, + path=path, + sequence_pattern=sequence_pattern, + glob_pattern=glob_pattern, + ) as cases: + results = {} + for case in cases: + results[case["name"]] = benchmark_call(case["callable"], iterations) + + if enable_profile: + profiles_dir.mkdir(parents=True, exist_ok=True) + for case in cases: + if case["profile_kind"] == "python": + profile_python_callable(case["callable"], case["name"], profiles_dir) + else: + profile_lss_subprocess( + case["path"], + case["name"], + profiles_dir, + glob_pattern=case.get("glob_pattern"), + ) + + return results + + +def build_run_payload( + sizes, + scenarios, + iterations, + enable_profile, + profiles_dir, + path=None, + sequence_pattern=None, + glob_pattern=None, +): + branch = detect_git_branch() + commit = detect_git_sha() + actual_profiles_dir = profiles_dir + if enable_profile and actual_profiles_dir is None: + actual_profiles_dir = make_default_profiles_dir(branch, commit) + + benchmarks = run_benchmarks( + sizes=sizes, + scenarios=scenarios, + iterations=iterations, + enable_profile=enable_profile, + profiles_dir=actual_profiles_dir, + path=path, + sequence_pattern=sequence_pattern, + glob_pattern=glob_pattern, + ) + + return { + "kind": "benchmark", + "branch": branch, + "commit": commit, + "repo_root": str(REPO_ROOT), + "lib_root": str(LIB_ROOT), + "script_path": str(SCRIPT_PATH), + "python_executable": sys.executable, + "pyseq_file": getattr(pyseq, "__file__", None), + "pyseq_seq_file": getattr(pyseq_seq, "__file__", None), + "python": platform.python_version(), + "platform": platform.platform(), + "timestamp_utc": datetime.now(timezone.utc).isoformat(), + "sizes": sizes, + "scenarios": scenarios, + "iterations": iterations, + "path": str(Path(path).resolve()) if path else None, + "sequence_pattern": sequence_pattern, + "glob_pattern": glob_pattern, + "profiled": enable_profile, + "profiles_dir": str(actual_profiles_dir) if actual_profiles_dir else None, + "benchmarks": benchmarks, + } + + +def build_comparison(baseline, candidate): + names = sorted(set(baseline["benchmarks"]) & set(candidate["benchmarks"])) + comparisons = [] + for name in names: + before = baseline["benchmarks"][name] + after = candidate["benchmarks"][name] + comparisons.append( + { + "benchmark": name, + "baseline_median_s": before["median_s"], + "candidate_median_s": after["median_s"], + "delta_s": after["median_s"] - before["median_s"], + "delta_pct": pct_delta(after["median_s"], before["median_s"]), + } + ) + + deltas = [row["delta_pct"] for row in comparisons] + return { + "kind": "benchmark-compare", + "baseline": { + "branch": baseline.get("branch"), + "commit": baseline.get("commit"), + "repo_root": baseline.get("repo_root"), + "lib_root": baseline.get("lib_root"), + "script_path": baseline.get("script_path"), + "python_executable": baseline.get("python_executable"), + "pyseq_file": baseline.get("pyseq_file"), + "pyseq_seq_file": baseline.get("pyseq_seq_file"), + }, + "candidate": { + "branch": candidate.get("branch"), + "commit": candidate.get("commit"), + "repo_root": candidate.get("repo_root"), + "lib_root": candidate.get("lib_root"), + "script_path": candidate.get("script_path"), + "python_executable": candidate.get("python_executable"), + "pyseq_file": candidate.get("pyseq_file"), + "pyseq_seq_file": candidate.get("pyseq_seq_file"), + }, + "benchmarks": comparisons, + "summary": { + "median_delta_pct": median(deltas), + "max_regression_pct": max(deltas) if deltas else 0.0, + "max_improvement_pct": min(deltas) if deltas else 0.0, + "benchmark_count": len(comparisons), + }, + } + + +def enforce_regression_threshold(payload, threshold_pct, threshold_seconds=0.0): + regressions = [ + row + for row in payload["benchmarks"] + if row["delta_pct"] > threshold_pct and row["delta_s"] > threshold_seconds + ] + if regressions: + worst = max(regressions, key=lambda row: row["delta_pct"]) + raise SystemExit( + "Regression threshold exceeded: " + f"{len(regressions)} benchmark(s) slower than +{threshold_pct:.2f}% " + f"and +{threshold_seconds:.3f}s " + f"(worst: {worst['benchmark']} at {worst['delta_pct']:+.2f}%, " + f"{worst['delta_s']:+.6f}s)" + ) + + +def write_run_summary(payload, stream): + print("# pyseq benchmarks", file=stream) + print("", file=stream) + print( + f"Branch: `{payload['branch']}` Commit: `{payload['commit']}` Sizes: `{','.join(str(size) for size in payload['sizes']) if payload['sizes'] else 'custom'}` Iterations: `{payload['iterations']}`", + file=stream, + ) + print(f"Repo: `{payload['repo_root']}`", file=stream) + print(f"Lib: `{payload['lib_root']}`", file=stream) + print(f"Python: `{payload['python_executable']}`", file=stream) + if payload.get("scenarios"): + print( + f"Scenarios: `{','.join(payload['scenarios'])}`", + file=stream, + ) + if payload.get("pyseq_file"): + print(f"PySeq: `{payload['pyseq_file']}`", file=stream) + if payload.get("pyseq_seq_file"): + print(f"PySeq seq: `{payload['pyseq_seq_file']}`", file=stream) + if payload.get("path"): + print(f"Path: `{payload['path']}`", file=stream) + if payload.get("glob_pattern"): + print(f"Glob: `{payload['glob_pattern']}`", file=stream) + if payload.get("sequence_pattern"): + print(f"Sequence pattern: `{payload['sequence_pattern']}`", file=stream) + if payload["profiled"] and payload["profiles_dir"]: + print(f"Profiles: `{payload['profiles_dir']}`", file=stream) + print("", file=stream) + print("| Benchmark | Median (s) | Mean (s) | Min (s) | Max (s) |", file=stream) + print("| --- | ---: | ---: | ---: | ---: |", file=stream) + for name, result in payload["benchmarks"].items(): + print( + f"| `{name}` | {result['median_s']:.6f} | {result['mean_s']:.6f} | {result['min_s']:.6f} | {result['max_s']:.6f} |", + file=stream, + ) + + +def write_compare_summary(payload, stream): + print("# pyseq benchmark comparison", file=stream) + print("", file=stream) + print( + f"Baseline: `{payload['baseline']['branch']}` `{payload['baseline']['commit']}` Candidate: `{payload['candidate']['branch']}` `{payload['candidate']['commit']}`", + file=stream, + ) + if payload["baseline"].get("lib_root") or payload["candidate"].get("lib_root"): + print( + f"Baseline lib: `{payload['baseline'].get('lib_root')}` Candidate lib: `{payload['candidate'].get('lib_root')}`", + file=stream, + ) + if payload["baseline"].get("python_executable") or payload["candidate"].get( + "python_executable" + ): + print( + f"Baseline python: `{payload['baseline'].get('python_executable')}` Candidate python: `{payload['candidate'].get('python_executable')}`", + file=stream, + ) + if payload["baseline"].get("pyseq_file") or payload["candidate"].get("pyseq_file"): + print( + f"Baseline pyseq: `{payload['baseline'].get('pyseq_file')}` Candidate pyseq: `{payload['candidate'].get('pyseq_file')}`", + file=stream, + ) + if payload["baseline"].get("pyseq_seq_file") or payload["candidate"].get("pyseq_seq_file"): + print( + f"Baseline pyseq.seq: `{payload['baseline'].get('pyseq_seq_file')}` Candidate pyseq.seq: `{payload['candidate'].get('pyseq_seq_file')}`", + file=stream, + ) + print( + "Summary: " + f"`{payload['summary']['benchmark_count']}` benchmarks " + f"median delta `{payload['summary']['median_delta_pct']:+.2f}%` " + f"max regression `{payload['summary']['max_regression_pct']:+.2f}%`", + file=stream, + ) + print("", file=stream) + print( + "| Benchmark | Baseline Median (s) | Candidate Median (s) | Delta (s) | Delta (%) |", + file=stream, + ) + print("| --- | ---: | ---: | ---: | ---: |", file=stream) + for row in payload["benchmarks"]: + print( + f"| `{row['benchmark']}` | {row['baseline_median_s']:.6f} | {row['candidate_median_s']:.6f} | {row['delta_s']:.6f} | {row['delta_pct']:+.2f}% |", + file=stream, + ) + + +def dump_json(payload, path): + with open(path, "w", encoding="utf-8") as handle: + json.dump(payload, handle, indent=2, sort_keys=True) + handle.write("\n") + + +def load_json(path): + with open(path, "r", encoding="utf-8") as handle: + return json.load(handle) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--compare", + nargs=2, + metavar=("BASELINE_JSON", "CANDIDATE_JSON"), + help="Compare two benchmark JSON files instead of running benchmarks", + ) + parser.add_argument("--full", action="store_true", help="Run larger default sizes") + parser.add_argument( + "--sizes", + type=parse_sizes, + help="Comma-separated dataset sizes, for example: 100,1000,10000,50000", + ) + parser.add_argument( + "--scenarios", + type=parse_csv_list, + help="Comma-separated synthetic scenarios, for example: contiguous,mixed", + ) + parser.add_argument("--iterations", type=int, help="Override timing iteration count") + parser.add_argument("--json", dest="json_path", help="Write JSON output to a file") + parser.add_argument( + "--no-profile", + action="store_true", + help="Skip the separate profiling pass", + ) + parser.add_argument( + "--profiles-dir", + help="Directory for generated .pstats and text profile summaries", + ) + parser.add_argument( + "--path", + help="Benchmark an existing directory instead of synthetic datasets", + ) + parser.add_argument( + "--glob", + dest="glob_pattern", + help="Optional glob pattern for lss and file list filtering when using --path", + ) + parser.add_argument( + "--sequence-pattern", + help="Optional compressed sequence pattern for resolve_sequence when using --path", + ) + parser.add_argument( + "--fail-on-regression-pct", + type=float, + help="Exit non-zero when any compared benchmark regresses by more than this percentage", + ) + parser.add_argument( + "--fail-on-regression-s", + type=float, + default=0.0, + help="Require this minimum absolute regression in seconds when using --fail-on-regression-pct", + ) + args = parser.parse_args() + + if args.compare: + payload = build_comparison( + load_json(args.compare[0]), + load_json(args.compare[1]), + ) + if args.json_path: + dump_json(payload, args.json_path) + write_compare_summary(payload, sys.stdout) + if args.fail_on_regression_pct is not None: + enforce_regression_threshold( + payload, + args.fail_on_regression_pct, + args.fail_on_regression_s, + ) + return + + sizes = None if args.path else (args.sizes or (FULL_SIZES if args.full else DEFAULT_SIZES)) + scenarios = None if args.path else (args.scenarios or DEFAULT_SCENARIOS) + iterations = args.iterations or (FULL_ITERATIONS if args.full else DEFAULT_ITERATIONS) + profiles_dir = Path(args.profiles_dir) if args.profiles_dir else None + payload = build_run_payload( + sizes=sizes, + scenarios=scenarios, + iterations=iterations, + enable_profile=not args.no_profile, + profiles_dir=profiles_dir, + path=args.path, + sequence_pattern=args.sequence_pattern, + glob_pattern=args.glob_pattern, + ) + + if args.json_path: + dump_json(payload, args.json_path) + write_run_summary(payload, sys.stdout) + + +if __name__ == "__main__": + main() diff --git a/scripts/build_pages_site.py b/scripts/build_pages_site.py new file mode 100644 index 0000000..a9fdee4 --- /dev/null +++ b/scripts/build_pages_site.py @@ -0,0 +1,142 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2011-2025, Ryan Galloway (ryan@rsgalloway.com) +# + +"""Build a simple Jekyll-friendly docs site from repository markdown files.""" + +import argparse +import re +import shutil +from pathlib import Path + +LINK_PATTERNS = ( + (r"\(README\.md\)", "(index.html)"), + (r"\(docs/README\.md\)", "(docs/index.html)"), + (r"\(docs/([^)]+)\.md\)", r"(docs/\1.html)"), + (r"\(([^:)#]+)\.md\)", r"(\1.html)"), +) + + +def rewrite_links(content: str) -> str: + """Rewrite local markdown links for generated HTML output.""" + updated = content + for pattern, replacement in LINK_PATTERNS: + updated = re.sub(pattern, replacement, updated) + return updated + + +def extract_title(content: str, fallback: str) -> str: + """Extract the first markdown H1 title or use a fallback.""" + for line in content.splitlines(): + if line.startswith("# "): + return line[2:].strip() + return fallback + + +def wrap_markdown(content: str, title: str) -> str: + """Add minimal Jekyll front matter to markdown content.""" + return f"---\nlayout: default\ntitle: {title}\n---\n\n{content}" + + +def write_markdown_page(src: Path, dst: Path, fallback_title: str): + """Copy a markdown file into the site tree with front matter and fixed links.""" + content = src.read_text(encoding="utf-8") + title = extract_title(content, fallback_title) + content = rewrite_links(content) + dst.parent.mkdir(parents=True, exist_ok=True) + dst.write_text(wrap_markdown(content, title), encoding="utf-8") + + +def write_site_config(output_dir: Path): + """Write a minimal Jekyll config file.""" + config = """title: pyseq +description: Python library for numbered file sequences +theme: minima +markdown: kramdown +permalink: pretty +""" + (output_dir / "_config.yml").write_text(config, encoding="utf-8") + + +def write_benchmarks_page( + output_dir: Path, + benchmark_summary: Path = None, + benchmark_json: Path = None, +): + """Create a latest benchmark report page and copy JSON artifacts.""" + benchmarks_dir = output_dir / "benchmarks" + benchmarks_dir.mkdir(parents=True, exist_ok=True) + + sections = [ + "# Latest Benchmarks", + "", + "This page is generated automatically from the benchmark workflows.", + "", + ] + + if benchmark_summary and benchmark_summary.exists(): + sections.extend([benchmark_summary.read_text(encoding="utf-8").strip(), ""]) + + downloads = [] + if benchmark_json and benchmark_json.exists(): + shutil.copy2(benchmark_json, benchmarks_dir / "benchmark.json") + downloads.append("- [Benchmark JSON](benchmark.json)") + + if downloads: + sections.extend(["## Downloads", "", *downloads, ""]) + + page = wrap_markdown("\n".join(sections).rstrip() + "\n", "Latest Benchmarks") + (benchmarks_dir / "index.md").write_text(page, encoding="utf-8") + + +def build_site(args): + repo_root = Path(args.repo_root).resolve() + output_dir = Path(args.output).resolve() + + if output_dir.exists(): + shutil.rmtree(output_dir) + output_dir.mkdir(parents=True, exist_ok=True) + + write_site_config(output_dir) + + # Home page from the repository README. + write_markdown_page(repo_root / "README.md", output_dir / "index.md", "pyseq") + + # Docs pages. + docs_dir = repo_root / "docs" + for src in docs_dir.glob("*.md"): + if src.name == "README.md": + dst = output_dir / "docs" / "index.md" + fallback = "Docs" + else: + dst = output_dir / "docs" / src.name + fallback = src.stem.replace("-", " ").title() + write_markdown_page(src, dst, fallback) + + if (docs_dir / "assets").exists(): + shutil.copytree(docs_dir / "assets", output_dir / "docs" / "assets") + + cname = repo_root / "CNAME" + if cname.exists(): + shutil.copy2(cname, output_dir / "CNAME") + + write_benchmarks_page( + output_dir, + benchmark_summary=Path(args.benchmark_summary) if args.benchmark_summary else None, + benchmark_json=Path(args.benchmark_json) if args.benchmark_json else None, + ) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--repo-root", default=".") + parser.add_argument("--output", required=True) + parser.add_argument("--benchmark-summary") + parser.add_argument("--benchmark-json") + args = parser.parse_args() + build_site(args) + + +if __name__ == "__main__": + main() diff --git a/scripts/install_precommit_hook.py b/scripts/install_precommit_hook.py new file mode 100644 index 0000000..bc5a63b --- /dev/null +++ b/scripts/install_precommit_hook.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2011-2026, Ryan Galloway (ryan@rsgalloway.com) +# + +"""Install an optional git pre-commit hook for local quality checks.""" + +import stat +import sys +from pathlib import Path + +HOOK_TEMPLATE = """#!/usr/bin/env bash +set -euo pipefail + +ROOT={root} +PYTHON={python} + +cd "$ROOT" + +"$PYTHON" -m isort --check-only --diff lib/pyseq tests scripts +"$PYTHON" -m black --check lib/pyseq tests scripts +"$PYTHON" -m flake8 lib/pyseq tests scripts +"$PYTHON" -m pytest tests -q +""" + + +def main(): + repo_root = Path(__file__).resolve().parent.parent + git_dir = repo_root / ".git" + hooks_dir = git_dir / "hooks" + hook_path = hooks_dir / "pre-commit" + + if not git_dir.exists(): + raise SystemExit("No .git directory found; run this from a git checkout.") + + hooks_dir.mkdir(parents=True, exist_ok=True) + hook_path.write_text( + HOOK_TEMPLATE.format( + root=repr(str(repo_root)), + python=repr(sys.executable), + ), + encoding="utf-8", + ) + hook_path.chmod(hook_path.stat().st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) + print(f"Installed pre-commit hook at {hook_path}") + + +if __name__ == "__main__": + main() diff --git a/scripts/time_lss.sh b/scripts/time_lss.sh new file mode 100755 index 0000000..b17aa6f --- /dev/null +++ b/scripts/time_lss.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ $# -lt 1 || $# -gt 2 ]]; then + echo "Usage: scripts/time_lss.sh [runs]" >&2 + exit 1 +fi + +target_path=$1 +runs=${2:-20} + +if [[ ! -d .venv ]]; then + echo "Error: expected .venv in repo root: $(pwd)/.venv" >&2 + exit 1 +fi + +if [[ ! -d lib/pyseq ]]; then + echo "Error: expected to run from repo root containing lib/pyseq" >&2 + exit 1 +fi + +if ! [[ $runs =~ ^[0-9]+$ ]] || [[ $runs -lt 1 ]]; then + echo "Error: runs must be a positive integer" >&2 + exit 1 +fi + +echo "# repo: $(pwd)" +echo "# target: $target_path" +echo "# runs: $runs" +echo "# python: $(realpath .venv/bin/python3)" + +PYTHONPATH=lib .venv/bin/python3 - <<'PY' +import sys +import pyseq +import pyseq.seq +print(f"# pyseq: {pyseq.__file__}") +print(f"# pyseq.seq: {pyseq.seq.__file__}") +print(f"# sys.executable: {sys.executable}") +PY + +for run in $(seq 1 "$runs"); do + seconds=$( + { + /usr/bin/time -f '%e' \ + env PYTHONPATH=lib .venv/bin/python3 -m pyseq.lss "$target_path" \ + >/dev/null + } 2>&1 + ) + echo "$run $seconds" +done diff --git a/tests/files/negA.-0001.exr b/tests/files/negA.-0001.exr new file mode 100644 index 0000000..88355c3 --- /dev/null +++ b/tests/files/negA.-0001.exr @@ -0,0 +1 @@ +dummy frame -0001 diff --git a/tests/files/negA.-0002.exr b/tests/files/negA.-0002.exr new file mode 100644 index 0000000..959bfb0 --- /dev/null +++ b/tests/files/negA.-0002.exr @@ -0,0 +1 @@ +dummy frame -0002 diff --git a/tests/files/negA.0000.exr b/tests/files/negA.0000.exr new file mode 100644 index 0000000..cd1b05b --- /dev/null +++ b/tests/files/negA.0000.exr @@ -0,0 +1 @@ +dummy frame 0000 diff --git a/tests/files/negA.0001.exr b/tests/files/negA.0001.exr new file mode 100644 index 0000000..211d717 --- /dev/null +++ b/tests/files/negA.0001.exr @@ -0,0 +1 @@ +dummy frame 0001 diff --git a/tests/files/stepA.1001.exr b/tests/files/stepA.1001.exr new file mode 100644 index 0000000..715b1fd --- /dev/null +++ b/tests/files/stepA.1001.exr @@ -0,0 +1 @@ +dummy frame 1001 diff --git a/tests/files/stepA.1004.exr b/tests/files/stepA.1004.exr new file mode 100644 index 0000000..6544903 --- /dev/null +++ b/tests/files/stepA.1004.exr @@ -0,0 +1 @@ +dummy frame 1004 diff --git a/tests/files/stepA.1007.exr b/tests/files/stepA.1007.exr new file mode 100644 index 0000000..f3912ce --- /dev/null +++ b/tests/files/stepA.1007.exr @@ -0,0 +1 @@ +dummy frame 1007 diff --git a/tests/files/stepA.1010.exr b/tests/files/stepA.1010.exr new file mode 100644 index 0000000..d7bfe70 --- /dev/null +++ b/tests/files/stepA.1010.exr @@ -0,0 +1 @@ +dummy frame 1010 diff --git a/tests/test_cli_interrupts.py b/tests/test_cli_interrupts.py index c8207e3..6db691e 100644 --- a/tests/test_cli_interrupts.py +++ b/tests/test_cli_interrupts.py @@ -48,9 +48,7 @@ def test_copy_move_cli_main_handles_keyboard_interrupt( def test_srm_cli_main_handles_keyboard_interrupt(monkeypatch, capsys, tmp_path): monkeypatch.chdir(tmp_path) - monkeypatch.setattr( - sremove, "resolve_sequence_reference", _raise_keyboard_interrupt - ) + monkeypatch.setattr(sremove, "resolve_sequence_reference", _raise_keyboard_interrupt) monkeypatch.setattr("sys.argv", ["srm", "a.%04d.exr"]) assert sremove.main() == 1 diff --git a/tests/test_lss.py b/tests/test_lss.py index 6c679ab..c65ddbe 100644 --- a/tests/test_lss.py +++ b/tests/test_lss.py @@ -36,8 +36,8 @@ import os import subprocess import tempfile -import pytest +import pytest from conftest import get_installed_command lss_bin = get_installed_command("lss") @@ -95,3 +95,19 @@ def test_lss_stdin_input(lss_fixture): assert result.returncode == 0 out = result.stdout assert " 3 shot.%04d.exr [1-3]" in out + + +def test_lss_compact_extended_range_format(tmp_path): + """The %x directive should render compact stepped ranges.""" + for frame in (1001, 1004, 1007, 1010): + (tmp_path / f"stepA.{frame:04d}.exr").write_text("frame\n") + + result = subprocess.run( + [lss_bin, "-f", "%h%x%t", str(tmp_path)], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + + assert result.returncode == 0, result.stderr + assert "stepA.1001-1010x3.exr" in result.stdout diff --git a/tests/test_performance.py b/tests/test_performance.py new file mode 100644 index 0000000..8d11e69 --- /dev/null +++ b/tests/test_performance.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python +# +# Copyright (c) 2011-2025, Ryan Galloway (ryan@rsgalloway.com) +# + +"""Performance regression tests for pyseq.""" + +import time +import unittest + +from pyseq import Sequence, get_sequences +from pyseq import seq as pyseq +from pyseq import uncompress + + +class PerformanceTests(unittest.TestCase): + """Coarse-grained performance regression tests.""" + + def test_sequence_construction_10k_frames(self): + """Single large contiguous sequences should remain fast.""" + pyseq.strict_pad = False + files = ["file.%03d.jpg" % i for i in range(1, 10000)] + + start = time.perf_counter() + seq = Sequence(files) + elapsed = time.perf_counter() - start + + self.assertEqual(str(seq), "file.1-9999.jpg") + self.assertEqual(len(seq), 9999) + self.assertLess(elapsed, 0.5) + + def test_sparse_large_missing_range(self): + """Sparse huge ranges should not trigger catastrophic expansion.""" + pyseq.strict_pad = False + files = ["image-1.jpg", "image-1000.jpg", "image-50000000.jpg"] + + start = time.perf_counter() + seq = get_sequences(files)[0] + missing = seq._get_missing() + formatted = seq.format("%M") + elapsed = time.perf_counter() - start + + self.assertEqual(seq.frames(), [1, 1000, 50000000]) + self.assertEqual(len(missing), 2) + self.assertEqual(formatted, "[2-999, 1001-49999999, ]") + # Keep this intentionally loose so we catch catastrophic regressions + # without making CI timing-sensitive. + self.assertLess(elapsed, 1.0) + + def test_stepped_range_parse_and_format(self): + """Medium-sized stepped ranges should parse and format quickly.""" + pyseq.strict_pad = False + + start = time.perf_counter() + seq = uncompress("render.%04d.exr 1001-10000x3", fmt="%h%p%t %x") + rendered = seq.format("%x") + elapsed = time.perf_counter() - start + + self.assertEqual(seq.frames()[0], 1001) + self.assertEqual(seq.frames()[-1], 9998) + self.assertEqual(rendered, "1001-9998x3") + self.assertLess(elapsed, 0.5) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_pyseq.py b/tests/test_pyseq.py index b164b9a..73a9f4a 100644 --- a/tests/test_pyseq.py +++ b/tests/test_pyseq.py @@ -34,18 +34,21 @@ """ import os -import re import random -import unittest +import re import subprocess import sys -import time +import tempfile +import unittest +from contextlib import contextmanager sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from conftest import get_installed_command -from pyseq import Item, Sequence, diff, uncompress, get_sequences -from pyseq import SequenceError + +from pyseq import Item, Sequence, SequenceError, config, diff, get_sequences from pyseq import seq as pyseq +from pyseq import uncompress +from pyseq.util import resolve_sequence_reference pyseq.default_format = "%h%r%t" @@ -55,6 +58,21 @@ def assert_read_only_attribute_error(message): assert any(snippet in message for snippet in valid_snippets), message +@contextmanager +def enable_negative_frames(): + previous = os.environ.get("PYSEQ_ALLOW_NEGATIVE_FRAMES") + os.environ["PYSEQ_ALLOW_NEGATIVE_FRAMES"] = "1" + config.set_frame_pattern(config.PYSEQ_FRAME_PATTERN) + try: + yield + finally: + if previous is None: + os.environ.pop("PYSEQ_ALLOW_NEGATIVE_FRAMES", None) + else: + os.environ["PYSEQ_ALLOW_NEGATIVE_FRAMES"] = previous + config.set_frame_pattern(config.PYSEQ_FRAME_PATTERN) + + class ItemTestCase(unittest.TestCase): """tests the Item class""" @@ -157,6 +175,13 @@ def test_is_sibling_method_is_working_properly(self): self.assertTrue(item1.is_sibling(item2)) self.assertTrue(item2.is_sibling(item1)) + def test_is_sibling_method_with_negative_frames(self): + item1 = Item("render.-0002.exr") + item2 = Item("render.-0001.exr") + + self.assertTrue(item1.is_sibling(item2)) + self.assertTrue(item2.is_sibling(item1)) + class SequenceTestCase(unittest.TestCase): """tests the pyseq""" @@ -512,6 +537,36 @@ def test_uncompress_is_working_properly_8(self): self.assertEqual(96, len(seq8)) + def test_uncompress_extended_range_with_step(self): + seq = uncompress("render.%04d.exr 1001-1010x3", fmt="%h%p%t %x") + self.assertEqual(seq.frames(), [1001, 1004, 1007, 1010]) + self.assertEqual(seq.format("%x"), "1001-1010x3") + + def test_uncompress_extended_range_negative_frames(self): + with enable_negative_frames(): + seq = uncompress("render.%04d.exr -10--1x3", fmt="%h%p%t %x") + self.assertEqual(seq.frames(), [-10, -7, -4, -1]) + self.assertEqual(seq.format("%h%p%t %x"), "render.%04d.exr -10--1x3") + + def test_uncompress_extended_range_descending(self): + seq = uncompress("render.%04d.exr 10-1x3", fmt="%h%p%t %x") + self.assertEqual(seq.frames(), [1, 4, 7, 10]) + self.assertEqual( + [(item.name, item.frame) for item in seq], + [ + ("render.0001.exr", 1), + ("render.0004.exr", 4), + ("render.0007.exr", 7), + ("render.0010.exr", 10), + ], + ) + self.assertEqual(seq.format("%x"), "1-10x3") + + def test_format_extended_range_multiple_segments(self): + seq = uncompress("render.%04d.exr 1-10x3, 20-30x5, 42", fmt="%h%p%t %x") + self.assertEqual(seq.frames(), [1, 4, 7, 10, 20, 25, 30, 42]) + self.assertEqual(seq.format("%x"), "1-10x3,20-30x5,42") + def test_get_sequences_is_working_properly_1(self): """testing if get_sequences is working properly""" seqs = get_sequences("./files/") @@ -534,6 +589,7 @@ def test_get_sequences_is_working_properly_1(self): "fileA.1-3.jpg", "fileA.1-3.png", "file_02.tif", + "negA.-2-1.exr", "z1_001_v1.1-4.png", "z1_002_v1.1-4.png", "z1_002_v2.1-4.png", @@ -560,15 +616,118 @@ def test_get_sequences_is_working_properly_3(self): for seq, expected_result in zip(seqs, expected_results): self.assertEqual(expected_result, seq.format("%h%p%t %r")) + def test_resolve_sequence_reference_with_stepped_serialized_range(self): + pyseq.strict_pad = False + reference = "./tests/files/stepA.%04d.exr 1001-1010x3" + seq, dirname = resolve_sequence_reference(reference) + + self.assertEqual(dirname, "./tests/files") + self.assertEqual(seq.frames(), [1001, 1004, 1007, 1010]) + self.assertEqual(seq.format("%h%p%t %x"), "stepA.%d.exr 1001-1010x3") + + def test_resolve_sequence_reference_with_stepped_embedded_range(self): + pyseq.strict_pad = False + reference = "./tests/files/stepA.1001-1010x3.exr" + seq, dirname = resolve_sequence_reference(reference) + + self.assertEqual(dirname, "./tests/files") + self.assertEqual(seq.frames(), [1001, 1004, 1007, 1010]) + self.assertEqual(seq.format("%h%p%t %x"), "stepA.%d.exr 1001-1010x3") + + def test_resolve_sequence_reference_with_descending_serialized_range(self): + pyseq.strict_pad = False + reference = "./tests/files/stepA.%04d.exr 1010-1001x3" + seq, dirname = resolve_sequence_reference(reference) + + self.assertEqual(dirname, "./tests/files") + self.assertEqual(seq.frames(), [1001, 1004, 1007, 1010]) + self.assertEqual( + [(item.name, item.frame) for item in seq], + [ + ("stepA.1001.exr", 1001), + ("stepA.1004.exr", 1004), + ("stepA.1007.exr", 1007), + ("stepA.1010.exr", 1010), + ], + ) + + def test_resolve_sequence_reference_with_descending_embedded_range(self): + pyseq.strict_pad = False + reference = "./tests/files/stepA.1010-1001x3.exr" + seq, dirname = resolve_sequence_reference(reference) + + self.assertEqual(dirname, "./tests/files") + self.assertEqual(seq.frames(), [1001, 1004, 1007, 1010]) + self.assertEqual( + [(item.name, item.frame) for item in seq], + [ + ("stepA.1001.exr", 1001), + ("stepA.1004.exr", 1004), + ("stepA.1007.exr", 1007), + ("stepA.1010.exr", 1010), + ], + ) + + def test_get_sequences_with_negative_filenames(self): + with enable_negative_frames(): + pyseq.strict_pad = True + seq = get_sequences( + [ + "render.-0002.exr", + "render.-0001.exr", + "render.0000.exr", + "render.0001.exr", + ] + )[0] + + self.assertEqual(seq.frames(), [-2, -1, 0, 1]) + self.assertEqual(seq.format("%h%p%t %x"), "render.%04d.exr -2-1") + + def test_get_sequences_with_negative_fixture_files(self): + with enable_negative_frames(): + pyseq.strict_pad = True + seq = get_sequences("./tests/files/negA*")[0] + + self.assertEqual(str(seq), "negA.-2-1.exr") + self.assertEqual(seq.frames(), [-2, -1, 0, 1]) + self.assertEqual(seq.format("%h%p%t %x"), "negA.%04d.exr -2-1") + + def test_resolve_sequence_reference_with_negative_filenames(self): + with enable_negative_frames(): + pyseq.strict_pad = True + with tempfile.TemporaryDirectory() as tmpdir: + for frame in (-2, -1, 0, 1): + if frame < 0: + frame_text = f"-{abs(frame):04d}" + else: + frame_text = f"{frame:04d}" + with open(os.path.join(tmpdir, f"render.{frame_text}.exr"), "w") as handle: + handle.write("dummy frame") + + seq, dirname = resolve_sequence_reference( + os.path.join(tmpdir, "render.%04d.exr -2-1") + ) + + self.assertEqual(dirname, tmpdir) + self.assertEqual(seq.frames(), [-2, -1, 0, 1]) + self.assertEqual(seq.format("%h%p%t %x"), "render.%04d.exr -2-1") + + def test_resolve_sequence_reference_with_negative_fixture_files(self): + with enable_negative_frames(): + pyseq.strict_pad = True + seq, dirname = resolve_sequence_reference("./tests/files/negA.%04d.exr -2-1") + + self.assertEqual(dirname, "./tests/files") + self.assertEqual(seq.frames(), [-2, -1, 0, 1]) + self.assertEqual(seq.format("%h%p%t %x"), "negA.%04d.exr -2-1") + class LSSTestCase(unittest.TestCase): """Tests lss command""" def run_command(self, *args): """a simple wrapper for subprocess.Popen""" - process = subprocess.Popen( - args, stdout=subprocess.PIPE, universal_newlines=True - ) + process = subprocess.Popen(args, stdout=subprocess.PIPE, universal_newlines=True) stdout, _ = process.communicate() return stdout @@ -585,49 +744,35 @@ def test_lss_is_working_properly_1(self): result = self.run_command(self.lss, test_files) self.assertEqual( - """ 10 012_vb_110_v001.%04d.png [1-10] - 10 012_vb_110_v002.%04d.png [1-10] - 7 a.%03d.tga [1-3, 10, 12-14] - 1 alpha.txt - 5 bnc01_TinkSO_tx_0_ty_0.%04d.tif [101-105] - 5 bnc01_TinkSO_tx_0_ty_1.%04d.tif [101-105] - 5 bnc01_TinkSO_tx_1_ty_0.%04d.tif [101-105] - 5 bnc01_TinkSO_tx_1_ty_1.%04d.tif [101-105] - 4 file.%02d.tif [1-2, 98-99] - 1 file.info.03.rgb - 3 file01.%03d.j2k [1-2, 4] - 4 file01_%04d.rgb [40-43] - 4 file02_%04d.rgb [44-47] - 4 file%d.03.rgb [1-4] - 3 fileA.%04d.jpg [1-3] - 3 fileA.%04d.png [1-3] - 1 file_02.tif - 4 z1_001_v1.%d.png [1-4] - 4 z1_002_v1.%d.png [1-4] - 4 z1_002_v2.%d.png [9-12] -""", + ( + " 10 012_vb_110_v001.%04d.png [1-10]\n" + " 10 012_vb_110_v002.%04d.png [1-10]\n" + " 7 a.%03d.tga [1-3, 10, 12-14]\n" + " 1 alpha.txt \n" + " 5 bnc01_TinkSO_tx_0_ty_0.%04d.tif [101-105]\n" + " 5 bnc01_TinkSO_tx_0_ty_1.%04d.tif [101-105]\n" + " 5 bnc01_TinkSO_tx_1_ty_0.%04d.tif [101-105]\n" + " 5 bnc01_TinkSO_tx_1_ty_1.%04d.tif [101-105]\n" + " 4 file.%02d.tif [1-2, 98-99]\n" + " 1 file.info.03.rgb \n" + " 3 file01.%03d.j2k [1-2, 4]\n" + " 4 file01_%04d.rgb [40-43]\n" + " 4 file02_%04d.rgb [44-47]\n" + " 4 file%d.03.rgb [1-4]\n" + " 3 fileA.%04d.jpg [1-3]\n" + " 3 fileA.%04d.png [1-3]\n" + " 1 file_02.tif \n" + " 2 negA.-%04d.exr [1-2]\n" + " 2 negA.%04d.exr [0-1]\n" + " 4 stepA.%d.exr [1001, 1004, 1007, 1010]\n" + " 4 z1_001_v1.%d.png [1-4]\n" + " 4 z1_002_v1.%d.png [1-4]\n" + " 4 z1_002_v2.%d.png [9-12]\n" + ), result, ) -class PerformanceTests(unittest.TestCase): - """tests the performance of pyseq""" - - def test_performance_1(self): - """tests performance for single 10k frame sequence""" - files = ["file.%03d.jpg" % i for i in range(1, 10000)] - s = time.time() - seq = Sequence(files) - e = time.time() - total_time = e - s - print("time taken to create sequence: %s" % (total_time)) - self.assertEqual(str(seq), "file.1-9999.jpg") - self.assertEqual(len(seq), 9999) - # Keep a loose upper bound so this stays meaningful without flaking on - # slower CI runners or across Python versions. - self.assertLess(total_time, 0.5) - - class TestIssues(unittest.TestCase): """tests reported issues on github""" @@ -795,9 +940,7 @@ def get_range(frames): self.assertTrue(len(missing), 5000000) self.assertEqual(missing[0][0], 100000001) self.assertEqual(missing[0][-1], 499999999) - self.assertEqual( - seqs[0].format(), " 2 image-%09d-2048x2048.jpg [100000000, 500000000]" - ) + self.assertEqual(seqs[0].format(), " 2 image-%09d-2048x2048.jpg [100000000, 500000000]") self.assertEqual(seqs[0].format("%M"), "[100000001-499999999, ]") # high missing frame count test 3 (from the issue) @@ -901,9 +1044,7 @@ def test_issue_83(self): ] # test using default frame pattern - seqs1 = pyseq.get_sequences( - filenames, frame_pattern=config.DEFAULT_FRAME_PATTERN - ) + seqs1 = pyseq.get_sequences(filenames, frame_pattern=config.DEFAULT_FRAME_PATTERN) self.assertEqual(len(seqs1), 1) # test if a new file in the sequence is included diff --git a/tests/test_scopy.py b/tests/test_scopy.py index 250066a..7ae55d6 100644 --- a/tests/test_scopy.py +++ b/tests/test_scopy.py @@ -36,15 +36,26 @@ import os import subprocess import tempfile + import pytest +from conftest import get_installed_command import pyseq -from conftest import get_installed_command from pyseq.scopy import copy_sequence scopy_bin = get_installed_command("scopy") +def _signed_frame_name(frame): + return f"{frame:05d}" if frame < 0 else f"{frame:04d}" + + +def _negative_env(): + env = os.environ.copy() + env["PYSEQ_ALLOW_NEGATIVE_FRAMES"] = "1" + return env + + @pytest.fixture def sample_sequence(tmp_path): """Create a dummy sequence: test.0001.exr through test.0003.exr""" @@ -140,3 +151,26 @@ def test_scopy_cli_embedded_range_source_and_dest(tmp_path): assert os.path.exists(tmp_path / f"plate.{i:04d}.rgb") for i in range(20, 23): assert os.path.exists(tmp_path / f"beauty.{i:04d}.rgb") + + +def test_scopy_cli_negative_sequence_string_source_and_dest(tmp_path): + """Serialized negative frame ranges should resolve before copying.""" + for frame in range(-2, 2): + (tmp_path / f"negA.{_signed_frame_name(frame)}.exr").write_text("dummy frame") + + src = str(tmp_path / "negA.%04d.exr") + " -2-0" + dest = str(tmp_path / "negB.%04d.exr") + " 100-102" + + result = subprocess.run( + [scopy_bin, src, dest], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + env=_negative_env(), + ) + + assert result.returncode == 0, result.stderr + for frame in range(-2, 2): + assert os.path.exists(tmp_path / f"negA.{_signed_frame_name(frame)}.exr") + for frame in range(100, 103): + assert os.path.exists(tmp_path / f"negB.{frame:04d}.exr") diff --git a/tests/test_sdiff.py b/tests/test_sdiff.py index 2f3bef2..8b31f43 100644 --- a/tests/test_sdiff.py +++ b/tests/test_sdiff.py @@ -37,10 +37,11 @@ import os import subprocess import tempfile + import pytest +from conftest import get_installed_command import pyseq -from conftest import get_installed_command from pyseq.sdiff import diff_sequences sdiff_bin = get_installed_command("sdiff") diff --git a/tests/test_sfind.py b/tests/test_sfind.py index 3624617..cf823f5 100644 --- a/tests/test_sfind.py +++ b/tests/test_sfind.py @@ -36,8 +36,8 @@ import os import subprocess import tempfile -import pytest +import pytest from conftest import get_installed_command sfind_bin = get_installed_command("sfind") diff --git a/tests/test_smove.py b/tests/test_smove.py index bea5983..68051db 100644 --- a/tests/test_smove.py +++ b/tests/test_smove.py @@ -36,15 +36,26 @@ import os import subprocess import tempfile + import pytest +from conftest import get_installed_command import pyseq -from conftest import get_installed_command from pyseq.smove import move_sequence smv_bin = get_installed_command("smv") +def _signed_frame_name(frame): + return f"{frame:05d}" if frame < 0 else f"{frame:04d}" + + +def _negative_env(): + env = os.environ.copy() + env["PYSEQ_ALLOW_NEGATIVE_FRAMES"] = "1" + return env + + @pytest.fixture def sample_sequence(): """Fixture to create a temporary directory with a sequence of files.""" @@ -245,3 +256,27 @@ def test_smv_cli_embedded_range_source_and_dest(tmp_path): assert os.path.exists(tmp_path / "plate.0005.rgb") for i in range(2, 5): assert not os.path.exists(tmp_path / f"plate.{i:04d}.rgb") + + +def test_smv_cli_negative_sequence_string_source_and_dest(tmp_path): + """Serialized negative frame ranges should resolve before moving.""" + for frame in range(-2, 2): + (tmp_path / f"negA.{_signed_frame_name(frame)}.exr").write_text("dummy frame") + + src = str(tmp_path / "negA.%04d.exr") + " -2-0" + dest = str(tmp_path / "negB.%04d.exr") + " 100-102" + + result = subprocess.run( + [smv_bin, src, dest], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + env=_negative_env(), + ) + + assert result.returncode == 0, result.stderr + for frame in range(100, 103): + assert os.path.exists(tmp_path / f"negB.{frame:04d}.exr") + for frame in range(-2, 1): + assert not os.path.exists(tmp_path / f"negA.{_signed_frame_name(frame)}.exr") + assert os.path.exists(tmp_path / "negA.0001.exr") diff --git a/tests/test_srm.py b/tests/test_srm.py index 5ee56bb..a134f82 100644 --- a/tests/test_srm.py +++ b/tests/test_srm.py @@ -36,15 +36,26 @@ import os import subprocess import tempfile + import pytest +from conftest import get_installed_command import pyseq -from conftest import get_installed_command from pyseq.sremove import remove_sequence srm_bin = get_installed_command("srm") +def _signed_frame_name(frame): + return f"{frame:05d}" if frame < 0 else f"{frame:04d}" + + +def _negative_env(): + env = os.environ.copy() + env["PYSEQ_ALLOW_NEGATIVE_FRAMES"] = "1" + return env + + @pytest.fixture def sample_sequence(): """Fixture to create a temporary directory with a sequence of files.""" @@ -147,3 +158,24 @@ def test_srm_cli_embedded_range(tmp_path): assert os.path.exists(tmp_path / "plate.0005.rgb") for i in range(2, 5): assert not os.path.exists(tmp_path / f"plate.{i:04d}.rgb") + + +def test_srm_cli_negative_sequence_string(tmp_path): + """Serialized negative frame ranges should resolve before removal.""" + for frame in range(-2, 2): + (tmp_path / f"negA.{_signed_frame_name(frame)}.exr").write_text("dummy frame") + + src = str(tmp_path / "negA.%04d.exr") + " -2-0" + + result = subprocess.run( + [srm_bin, src], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + env=_negative_env(), + ) + + assert result.returncode == 0, result.stderr + for frame in range(-2, 1): + assert not os.path.exists(tmp_path / f"negA.{_signed_frame_name(frame)}.exr") + assert os.path.exists(tmp_path / "negA.0001.exr") diff --git a/tests/test_sstat.py b/tests/test_sstat.py index 0611a1f..82b8e15 100644 --- a/tests/test_sstat.py +++ b/tests/test_sstat.py @@ -33,14 +33,15 @@ Contains tests for the sstat module. """ +import json import os import subprocess import tempfile -import json + import pytest +from conftest import get_installed_command import pyseq -from conftest import get_installed_command from pyseq.sstat import json_sstat sstat_bin = get_installed_command("sstat") diff --git a/tests/test_stree.py b/tests/test_stree.py index 791782d..4f5089e 100644 --- a/tests/test_stree.py +++ b/tests/test_stree.py @@ -36,8 +36,8 @@ import os import subprocess import tempfile -import pytest +import pytest from conftest import get_installed_command stree_bin = get_installed_command("stree")