diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..945eef9 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,18 @@ +* text=auto eol=lf + +# Keep development-only material out of the distributed package. +/.github export-ignore +/.githooks export-ignore +/scripts export-ignore +/tests export-ignore +/workbench export-ignore +/.gitattributes export-ignore +/.gitignore export-ignore +/phpstan.neon.dist export-ignore +/phpunit.xml.dist export-ignore +/pint.json export-ignore +/testbench.yaml export-ignore + +# Diff/linguist hints +*.php diff=php +/tests/** linguist-vendored diff --git a/.githooks/commit-msg b/.githooks/commit-msg new file mode 100755 index 0000000..07b0ad9 --- /dev/null +++ b/.githooks/commit-msg @@ -0,0 +1,64 @@ +#!/bin/bash +# +# Enforce single-line commit messages with no attribution trailers. +# +# Repository reporting and per-developer exposure coverage are derived from git +# history. Co-author trailers split authorship across two identities and skew +# those reports; multi-line bodies are noise the parser has to strip. The "why" +# belongs in the PR description, the CHANGELOG, or a comment at the code site, +# where it stays readable. +# +# Installed by `composer setup-hooks`, which points core.hooksPath at .githooks. + +MSG_FILE="$1" +SOURCE="$2" + +# Merges, squashes and reverts generate bodies git wrote itself. +case "$SOURCE" in + merge|squash) exit 0 ;; +esac + +# Strip comments and trailing blank lines; that is what git will actually store. +BODY="$(grep -v '^#' "$MSG_FILE" | sed -e :a -e '/^\s*$/{$d;N;ba' -e '}')" + +if [[ -z "${BODY//[[:space:]]/}" ]]; then + # An empty message aborts the commit anyway; let git say so. + exit 0 +fi + +FIRST_LINE="$(printf '%s\n' "$BODY" | head -n 1)" + +case "$FIRST_LINE" in + Revert\ \"*) exit 0 ;; +esac + +fail() { + echo "" + echo "🚫 Commit rejected: $1" + echo "" + echo " Commit messages must be a single line, with no body and no trailers." + echo " Put the reasoning in the PR description, the CHANGELOG, or a code comment." + echo "" + echo " Got:" + printf '%s\n' "$BODY" | sed 's/^/ | /' + echo "" + exit 1 +} + +if printf '%s\n' "$BODY" | grep -qiE '^[[:space:]]*(co-authored-by|claude-session|signed-off-by[[:space:]]*:[[:space:]]*claude)'; then + fail "attribution trailers break authorship reporting." +fi + +if printf '%s\n' "$BODY" | grep -qiE 'claude\.ai/code/session'; then + fail "session links do not belong in git history." +fi + +if [[ "$(printf '%s\n' "$BODY" | wc -l | tr -d ' ')" -gt 1 ]]; then + fail "the message has more than one line." +fi + +if [[ ${#FIRST_LINE} -gt 72 ]]; then + fail "the subject is ${#FIRST_LINE} characters; keep it to 72." +fi + +exit 0 diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 30c8a49..d159e42 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -9,4 +9,10 @@ updates: schedule: interval: "weekly" labels: - - "dependencies" \ No newline at end of file + - "dependencies" + - package-ecosystem: "composer" + directory: "/" + schedule: + interval: "weekly" + labels: + - "dependencies" diff --git a/.github/workflows/php-tests.yml b/.github/workflows/php-tests.yml index 193f986..4506dd5 100644 --- a/.github/workflows/php-tests.yml +++ b/.github/workflows/php-tests.yml @@ -10,19 +10,18 @@ jobs: test: runs-on: ${{ matrix.os }} strategy: - fail-fast: true + # One unusable matrix leg should not cancel the other nine. + fail-fast: false matrix: os: [ubuntu-latest] - php: [8.3, 8.4] - laravel: [11.*, 12.*] + php: ['8.3', '8.4', '8.5'] + laravel: [12.*, 13.*] stability: [prefer-lowest, prefer-stable] include: - - laravel: 11.* - testbench: ^9.9 - carbon: ^2.63 - laravel: 12.* testbench: 10.* - carbon: ^2.63|^3.0 + - laravel: 13.* + testbench: 11.* name: P${{ matrix.php }} - L${{ matrix.laravel }} - ${{ matrix.stability }} - ${{ matrix.os }} @@ -42,13 +41,40 @@ jobs: echo "::add-matcher::${{ runner.tool_cache }}/php.json" echo "::add-matcher::${{ runner.tool_cache }}/phpunit.json" + # Carbon is not pinned here: the package only reaches it through + # Laravel's helpers, so whatever the framework resolves is the version + # worth testing against. - name: Install dependencies run: | - composer require "laravel/framework:${{ matrix.laravel }}" "orchestra/testbench:${{ matrix.testbench }}" "nesbot/carbon:${{ matrix.carbon }}" --no-interaction --no-update + composer require "laravel/framework:${{ matrix.laravel }}" "orchestra/testbench:${{ matrix.testbench }}" --no-interaction --no-update composer update --${{ matrix.stability }} --prefer-dist --no-interaction - name: List Installed Dependencies run: composer show -D - name: Execute tests - run: vendor/bin/pest --ci --bail --compact --memory --coverage \ No newline at end of file + run: vendor/bin/pest --ci --compact --memory --coverage --min=100 + + performance: + name: performance guards + runs-on: ubuntu-latest + timeout-minutes: 10 + + steps: + - uses: actions/checkout@v4 + + - name: Setup PHP + uses: shivammathur/setup-php@v2 + with: + php-version: '8.4' + extensions: dom, curl, libxml, mbstring, zip, pcntl, pdo, sqlite, pdo_sqlite, bcmath, intl, fileinfo + # Explicitly no coverage: instrumentation dominates the clock and + # flattens the difference between a fast and a slow implementation, + # so these assertions skip themselves when a driver is active. + coverage: none + + - name: Install composer dependencies + uses: ramsey/composer-install@v3 + + - name: Run performance guards + run: vendor/bin/pest --testsuite=Performance --ci diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml new file mode 100644 index 0000000..3563961 --- /dev/null +++ b/.github/workflows/security.yml @@ -0,0 +1,61 @@ +name: Security + +# SAST and dependency advisories, on a schedule as well as on push so an +# advisory published while nobody is committing is still caught. +on: + push: + branches: [main] + pull_request: + schedule: + - cron: '30 5 * * 1' + workflow_dispatch: + +permissions: + contents: read + +jobs: + semgrep: + name: Semgrep SAST + runs-on: ubuntu-latest + timeout-minutes: 10 + container: + image: semgrep/semgrep + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + + # The package's own rule samples and test fixtures are credential-shaped + # by design; the Stripe sample is Stripe's public documentation key. + # That rule is excluded rather than the files, so every other secret + # rule still runs over them. + - name: Semgrep scan + run: > + semgrep scan + --config p/php + --config p/secrets + --exclude-rule generic.secrets.security.detected-stripe-api-key.detected-stripe-api-key + --error + --text + + audit: + name: Dependency advisories + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + + - name: Setup PHP + uses: shivammathur/setup-php@v2 + with: + php-version: '8.4' + tools: composer:v2 + coverage: none + + - name: Install dependencies + run: composer install --no-interaction --prefer-dist --no-progress + + - name: Composer audit + run: composer audit diff --git a/.github/workflows/static-analysis.yml b/.github/workflows/static-analysis.yml index dfcbae4..7044652 100644 --- a/.github/workflows/static-analysis.yml +++ b/.github/workflows/static-analysis.yml @@ -4,9 +4,17 @@ on: push: paths: - '**.php' - - 'composer.lock' + - 'composer.json' - 'phpstan.neon.dist' - - '.github/workflows/phpstan.yml' + - 'rector.php' + - '.github/workflows/static-analysis.yml' + pull_request: + paths: + - '**.php' + - 'composer.json' + - 'phpstan.neon.dist' + - 'rector.php' + - '.github/workflows/static-analysis.yml' jobs: phpstan: @@ -19,12 +27,15 @@ jobs: - name: Setup PHP uses: shivammathur/setup-php@v2 with: - php-version: '8.4' + php-version: '8.5' extensions: dom, curl, libxml, mbstring, zip, pcntl, pdo, sqlite, pdo_sqlite, bcmath, soap, intl, gd, exif, iconv, imagick, fileinfo, swoole, openssl coverage: none - name: Install composer dependencies uses: ramsey/composer-install@v3 + - name: Run Rector (dry run) + run: ./vendor/bin/rector process --dry-run --no-progress-bar + - name: Run PHPStan - run: ./vendor/bin/phpstan --error-format=github \ No newline at end of file + run: ./vendor/bin/phpstan analyse --error-format=github --no-progress --memory-limit=1G \ No newline at end of file diff --git a/.github/workflows/style-check.yml b/.github/workflows/style-check.yml index 981eedd..58c07c6 100644 --- a/.github/workflows/style-check.yml +++ b/.github/workflows/style-check.yml @@ -2,6 +2,7 @@ name: Code Style on: workflow_dispatch: + pull_request: push: branches-ignore: - 'dependabot/npm_and_yarn/*' @@ -14,14 +15,11 @@ jobs: - name: Setup PHP uses: shivammathur/setup-php@v2 with: - php-version: 8.3 + php-version: '8.5' - name: Checkout uses: actions/checkout@v4 - - name: Copy .env - run: php -r "file_exists('.env') || copy('.env.example', '.env');" - - name: Install Dependencies run: composer install -q --no-ansi --no-interaction --no-scripts --no-progress --prefer-dist diff --git a/.gitignore b/.gitignore index 28a6c42..c3002b5 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,8 @@ vendor composer.lock node_modules build +.phpunit.cache +infection.log .pint.cache .idea .DS_Store diff --git a/.semgrepignore b/.semgrepignore new file mode 100644 index 0000000..03b3b65 --- /dev/null +++ b/.semgrepignore @@ -0,0 +1,7 @@ +# Dependencies and generated output; findings there belong upstream. +vendor/ +node_modules/ +.phpunit.cache/ + +# Test fixtures hold deliberately fake credentials the scanner must find. +tests/Feature/fixtures/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 6e8133a..23cf7e8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,405 @@ All notable changes to this project will be documented in this file. +## v1.0.0 - 2026-09-14 + +Hardening and completeness passes across correctness, security, performance, +packaging and conventions. Each item is one commit, with tests. + +### Added - capability + +- **Path rules.** `'request.headers.authorization' => 'redact'` names a location + outright, with `*` for one level, `**` for any depth and `users[*].token` for + lists. Checked first and, when one matches, instead of everything else - no key + matching, no pattern scanning, no walk below the node. Compiled once into a + trie walked in lockstep with the payload, so 200 rules cost about what one + does. The more specific pattern always wins, so declaration order never + matters. +- **Operators, separated from detection.** `redact`, `mask`, `partial`, + `remove`, `hash`, `surrogate` and `preserve`, chosen per entity rather than + per pattern. Register your own with `Redactor::registerOperator()`. Detection + now says only what was found and where; what happens to it is a separate, + configurable decision. +- **Deterministic pseudonymisation.** `surrogate` and `hash` replace a value + with a stable stand-in, so the same email always yields the same output and + redacted logs stay joinable - counts, joins and traces all survive. Surrogates + preserve shape: an email stays a valid email, a card stays Luhn-valid with its + BIN, and anything else keeps its character classes and separators. One-way + (HMAC, not encryption); falls back to plain redaction when no key is + available, rather than emitting an unkeyed stand-in that would look joinable + and silently not be. +- **Confidence scoring.** Detections carry a score and the signals behind it, so + a profile is tuned with one `min_confidence` number instead of by weakening + patterns. A passing checksum or a nearby credential keyword raises the score, + which lets the same pattern be filtered as noise alone and reported when + corroborated. Surfaced in scan output, mapped onto SARIF levels, and filterable + with `--min-confidence`. +- **Streaming file scanning.** Files are read as overlapping windows of lines, + so memory stays flat whatever the size. Windows overlap so a secret spanning a + boundary is still found; duplicates are dropped by fingerprint. +- **Credential verification.** `--verify` asks each provider whether a detected + credential is live, ranking confirmed-live findings above everything else. + Off unless config enables it, the run passes `--verify`, and the provider is + on an explicit allowlist - and never reachable from the redaction path at all. + The command names every host before contacting any. The secret never reaches + a finding, so it cannot escape through JSON, SARIF or a baseline. +- **`observability` profile**, set up to pseudonymise rather than redact. +- **Provider credentials in every profile.** JWTs, bearer tokens, PEM private + key blocks, credential URLs for any scheme, and AWS, GitHub, Stripe, Slack, + OpenAI, Anthropic, Google and SendGrid keys were only recognised by the + `file_scan` profile; the `default`, `strict` and `observability` profiles + relied on entropy, which misses a 20-character AWS key outright and a GitHub + token by a tenth of a bit. The rules are defined once at the top of the + config and spread into each profile. +- **Identity patterns that match what people actually write.** Non-ASCII + emails, IBANs in the spaced form banks print, international and E.164 phone + numbers. A bare ten-digit run is only a phone number next to a label such as + `phone` or `tel`; before, every Unix timestamp and ten-digit order number in + a log message was redacted as one. The `strict` profile's phone rule, which + matched any run of seven digits and spaces, is gone. +- **Pattern `keywords` and `min_length`.** A rule can name literals that must + appear in the value before its pattern is tried - a prefilter for cost, since + `['@']` keeps the email regex off almost every string in a payload, and for + precision, so a bare ten-digit run needs a `phone` label somewhere before it + is believed - and the shortest text it could match, so a shorter value skips + the rule with one integer compare. Every shipped rule declares both where + they apply. +- **Allow-lists.** A profile `allowlist` of literals and regexes that are + never findings whichever detector reports them, and a per-rule `allow` list + scoped to one rule. Checked after detection, so patterns stay as strong as + written. An entry that cannot be evaluated allows nothing. +- **Dictionary rules.** A pattern can be a `words` list - codenames, customer + names, anything no regex expresses - compiled into one whole-word, + case-insensitive alternation, longest first. +- **Known secrets.** `known_secrets.values` and `known_secrets.config` register + the application's own credentials - `app.key` by default - so they are + redacted wherever they appear verbatim, in every profile; the config form + registers every string under an array key. `Redactor::registerSecret()` adds + one at runtime. Values under eight characters are refused. +- **`redactor:allow` on a line** suppresses the scanner's findings for that + line, for the fixture or the documented example that a baseline would also + accept but without the reason living in a JSON file. +- **Entity recognition.** `EntityRecognitionStrategy` asks a named entity + recogniser about free text and feeds what it finds - people, places, + organisations - through the same overlap resolution, confidence floor and + operators as every other detector. The built-in driver speaks Presidio's + `/analyze` contract; `Redactor::registerRecognizer()` adds others. Gated to + prose-shaped values within a length band, to the labels asked for and the + score threshold set; every span's character offsets are converted and + verified against the value before replacement; failures degrade to + rules-only and a circuit breaker stops a dead sidecar being asked on every + log line. Present in the shipped profiles and inert until enabled. +- **Git-aware scanning.** `redactor:scan --staged`, `--diff=` and + `--history[=]` scan only the lines a change adds, on their real line + numbers, with the commit that added them for history. A secret removed by a + later commit is still found. Paths given with a git mode act as a pathspec; + exclude patterns still apply. +- **`redact` middleware.** `->middleware('redact:profile')` redacts a + response before it is sent: JSON as data, text as text, files untouched, + never writing `_redacted` markers into a payload, failing closed to a 500. +- **MCP and AI adapters.** `Mcp\RedactsResponses` on a Laravel MCP server + redacts tool results, structured content, resource reads, prompt messages, + streamed output and errors over any transport; `Ai\RedactPrompt` middleware + redacts a prompt before the provider and resolves tokens in the answer. Both + SDKs are suggested, not required. +- **`RedactionPerformed` event** with the profile, keys and counts per rule + and entity, never a value; a throwing listener cannot break redaction. +- **Reversible tokens.** The `tokenize` operator replaces a value with a + stable, model-friendly token (`tok_email_k4m9rp2xzq`) and keeps the original + encrypted in the cache for a TTL; `Redactor::detokenize()` exchanges known + tokens back and leaves unknown ones alone. `Tokenization\TokenStore` is the + contract for another backing store. +- **Streaming redaction.** `Streaming\StreamRedactor` redacts chunk by chunk + with a hold-back window, so a secret split across two chunks is still caught; + `through()` for iterables of chunks, `wrap()` for echoing callbacks, + `response()` for a streamed response. The `redact` middleware applies it to + streamed responses automatically. +- **`nullify` operator.** Replaces a value with null so a typed field keeps + its type and its key; inside a string it deletes the span. +- **`Redactor::fake()`** for application test suites: a redactor that still + redacts but records every call, with `assertNeverEmitted()`, + `assertRedacted()`, `assertFinding()`, `assertProfileUsed()` and friends, + so a test can prove a secret never left rather than hope it did not. +- **Self-testing rules.** A rule carries `samples` and `counter_samples`, and + `redactor:validate` runs them through the real detection path - keywords, + min_length, validators and allow-lists applied - failing on a rule that no + longer detects a sample or detects a counter-sample. Every shipped rule has + both. +- **Ruleset fingerprint.** A digest of the rules a scan ran, reported in JSON + and SARIF and recorded in baselines; a baseline made under a different + ruleset is warned about. +- **Decoding in the scanner.** Base64 tokens, percent-encoded runs and + JSON-escaped lines are decoded one layer deep and scanned; a finding names + the encoding and its excerpt comes from the decoded, redacted text. + `scan.decode` switches it off. +- **JUnit output** (`--output=junit`), a publishable pre-commit hook and a + GitHub workflow (`vendor:publish --tag=redactor-ci`) that scans pull + requests over their base and uploads SARIF. +- **`Detector` contract.** Anything that can report `Detection`s against a + string - a regex, an entropy measure, a recogniser model in another process + - plugs into the same resolution and operator pipeline. + +- `Redactor::inspect()` returning a `RedactionResult`. (R-07) +- `Redactor::redactSafely()`, which never throws. (R-04) +- `php artisan redactor:validate` - resolves every profile and fails on the + broken ones, including keys listed as both safe and blocked. (R-04, R-02) +- Pattern rules: `mode` (replace/mask/partial/remove/full), `keep`, + `mask_character`, `capture` and `validator`. (R-01, R-16, R-17) +- `max_depth` and `shannon_entropy.charset_thresholds` profile settings. + (R-03, R-16) +- `redactor:scan --output=sarif` for GitHub code scanning, and + `--baseline` / `--update-baseline` so CI fails only on new findings. (R-10) + +### Performance + +- Compiled key matchers are resolved once with the profile instead of being + looked up per call. KeyMatcher memoises on the pattern list, so finding the + cached matcher meant building an implode() of every configured key on every + check - 0.203us against the 0.050us match it was avoiding, making the cache + cost four times what it saved. +- Entropy analysis is skipped for values shorter than min_length. A byte count + is an upper bound on a character count, so the check can only skip work that + was provably going to find nothing; most values in a log payload are well + under the threshold. Measured at 40x over a realistic value set. +- Tokenising uses the non-/u patterns for ASCII subjects. The /u modifier makes + PCRE validate the whole subject as UTF-8 on every call: 40us against 12us to + split a 2.2KB string. The choice is made per subject, because dropping /u for + non-ASCII input would join tokens rather than merely run faster. +- Matched spans are rewritten in a single left-to-right pass. substr_replace + builds a whole new string per replacement, so a value with several matches + copied it several times. +- An unchanged subtree is returned as it arrived rather than rebuilt, so a + payload that redacts to nothing costs a walk and no copy. +- Net effect, measured on PHP 8.5.8 with opcache and the PCRE JIT off + (production is faster in absolute terms): a 31-leaf request payload through + the `default` profile costs 68us against 48us before this work, with the + profile now running 20 detection rules instead of 6, scrubbing the + application's own secrets and scoring entropy hits; a Monolog record through + the processor 84us against 59us; a 1 MB free-text subject 24ms against 38ms + with peak memory down from 10 MB to 1 MB; a 64 KB subject with twenty + secrets 2.8ms against 4.7ms. Scaling is linear in both dimensions. A + profile that wants the old cost back removes the rules it does not need. +- Hot-path configuration reads go through a repository held per container + instance rather than the `config()` helper, which resolves the repository + through the container on every call and had added 2-4us to every redaction. +- The pseudonymizer is resolved lazily by the operators that need it. Routing + blocked keys through operators had made every redaction with a blocked key + derive an HMAC key it then never used. +- Each rule asks a capture-free `preg_match()` before `preg_match_all()` with + offsets, since most rules do not match most values, and skips the subject + outright when it is shorter than the rule's `min_length`. +- The entropy detector asks PCRE for tokens of at least `min_length` rather + than every token, since shorter ones can never qualify. A 1 MB subject of + ordinary words held ~180,000 [token, offset] pairs - ten times the input - + and now holds none. +- Resolved profiles are cached and invalidated by comparing the raw config, so + `fromConfig()` no longer revalidates every pattern and recompiles the path + trie on every redaction: 0.2285ms -> 0.0011ms for a profile with 200 path + rules, and flat with rule count rather than linear. + +- Blocked-key matching compiles its pattern list once instead of rebuilding a + regex per key per call: 1.223 us -> 0.288 us per check. (R-12) +- Nested nodes are dispatched through the strategy chain once rather than + twice. (R-13) +- `Redactor` and `Scanner` are container singletons, so the strategy cache + survives. (R-11) +- Net effect on the default profile: ~15,000 -> ~21,000 redactions/sec, while + doing strictly more work than before. + +### Changed - conventions, following Laravel's first-party packages + +- **In-process recognition.** The companion package + `kirschbaum-development/redactor-onnx` registers an `onnx` driver that runs + a token classification model through TransformersPHP with no sidecar. +- **Recogniser batching.** Every prose value in a payload is recognised in + one call before the walk, so a record with fifty free-text fields costs one + round trip. `BatchRecognizer` for recognisers that take a list, and + `PrimingStrategy` for any strategy that pays per call rather than per value. +- **Four more verifiers.** OpenAI, Anthropic, SendGrid and Google API keys, + each behind the same three gates; `SecretVerifier::register()` adds your own. +- **Region packs.** National identifiers and VAT numbers for GB, NL, DE, FR, + IT, ES, BE, SE, NO, CA, AU and the rest of the EU, each with its checksum + (NHS mod-11, BSN eleven-proof, Steuer-ID, NIR key, DNI letter, codice + fiscale, Belgian mod-97, personnummer, fødselsnummer, SIN, TFN, VAT by + country), keywords where the shape is common, and samples that + `redactor:validate` proves. Switched on per profile with `regions`. +- **Custom validators.** `Validator::extend('name', fn)` makes a validator + usable from any rule; an unknown validator name is now a configuration error. +- **Per-call entity filtering.** `Redactor::profile('x')->only(['email'])` and + `->except([...])` act on a subset of what the profile can find, for the + export that only needs two things hidden. A key rule's entity is the key + name, a path rule's the key it lands on; fail-closed detections are never + filtered out. +- **Fluent entry point.** `Redactor::profile('strict')->withoutMarkers()->redact($data)` + and `->inspect($data)`; `Redactor::inspect()` returns the result with its + findings. `inspect()` remains. The redactor and the pending + redaction are `Macroable` and `Conditionable`. +- **Package exceptions.** Everything thrown implements + `Exceptions\RedactorException`: `ConfigurationException` and + `ProfileNotFoundException` (both still `InvalidArgumentException`), + `PseudonymizationKeyException`, `GitException`. Messages name the offending + value in `[brackets]`. +- **Results are `Arrayable` and `JsonSerializable`.** `RedactionResult`, + `MatchFinding` and `ScanFinding`; a finding's array form omits the matched + text. +- **Renamed, without aliases.** `ReadactFormatter` is `Logging\RedactorFormatter`, + `CustomLogTap` is `Logging\RedactorFormatterTap`, + `RedactionStrategyInterface` is `Strategies\Contracts\Strategy`, + `redactWithMetadata()` is `inspect()`, and `getAvailableProfiles()`, + `profileExists()` and `getStrategies()` are `profiles()`, `hasProfile()` + and `strategies()`. The old names are gone rather than deprecated. +- Services are no longer `final`; value objects stay `final readonly`. + Configuration, events and the container are reached the way first-party + packages reach them, and the service provider registers commands and + publishing in `boot()` behind `runningInConsole()`. + +### Changed - behaviour you should read before upgrading + +- **Detections are collected and the value rewritten once.** The regex and + entropy strategies no longer rewrite the string as they go; they report, the + context resolves overlaps and applies the confidence floor, and the original + value is rewritten in one pass. Of two overlapping reports the higher score + wins, then the rule listed first. Findings for a `preserve` operator are now + reported without marking the payload redacted. +- **The pseudonymisation salt is shared across profiles.** It defaulted to the + profile name, so two channels on different profiles produced different + surrogates for the same user and could not be joined. Set a profile's own + `pseudonymization.salt` to break correlation on purpose. +- **Blocked keys go through operators.** The key name is the entity, so + `operators.email` applies to a value under an `email` key. Findings from a + blocked key now carry the value they matched and a certain score. +- **Long strings are truncated and scanned, not replaced.** A value over + `max_value_length` keeps its head, which the remaining strategies still + inspect, followed by `[REDACTED] (String truncated: 65536 characters, 5000 + kept)`. The values most often over the limit in a Laravel log are stack traces + and request bodies, and replacing them wholesale destroyed exactly what the + reader needed. `large_string_behavior: redact` restores the old behaviour. +- **Throwables, dates, enums and closures pass through the walk untouched.** A + Throwable has no public properties and encoded to `{}`, so `['exception' => + $e]` reached the formatter as `[]` and the stack trace was lost; a Carbon + instance was exploded into its `toArray()` components. Key rules still apply + to these values, so `['secret' => $enum]` is still redacted. +- **Redaction now replaces the matched span, not the whole value.** + `redact('User bob@example.com placed order 123')` returns + `'User [REDACTED] placed order 123'` rather than `'[REDACTED]'`. (R-01) +- **`safe_keys` preserves the entire subtree**, and the shipped profiles no + longer list `message`, `title`, `url`, `path`, `ip`, `user_agent`, `source` or + `target` as safe - all of them are free text or personal data, and with them + safe the values were emitted verbatim. `session_id` was listed as both safe + and blocked; it is now blocked only. (R-02) +- **Monolog integration moved to a processor.** Use + `Logging\RedactorTap` / `Logging\RedactorProcessor`, which redact message, + context and extra without touching the channel's output format. + `RedactorFormatter` still works and can now wrap an inner formatter. (R-06) +- **Scan findings are structured**: rule, line, column and a redacted excerpt, + instead of one opaque `full_content_redacted` record per file. (R-10) +- **Removed** `Redactor::addStrategy()`, `removeStrategy()`, + `calculateShannonEntropy()` and `isCommonPattern()`. The two useful ones are + now public on `ShannonEntropyStrategy`. (R-20) +- Invalid configuration now throws with the offending path named, instead of + silently falling back to a default. (R-09) + +### Fixed - correctness and security + +- A surrogate written by the regex strategy was re-detected by the entropy + strategy that ran next - it has the same shape and entropy as the value it + replaced - and turned into `[REDACTED]`, destroying the joinability the + profile paid for. Detectors now all see the original value. +- The scanner reported the wrong column for the second finding on a line: each + rule measured its offsets against the string the previous rule had already + rewritten. Offsets are now always against the original. +- Entropy detections bypassed `operators`, `min_confidence` and confidence + scoring entirely, and the scanner ranked their null score as `high` - above a + Luhn-validated card. They now carry a score and go through the same policy. +- The scanner skipped every text file in a legacy encoding as binary: the + printable-ratio check ran a Unicode regex over a sample it already knew was + not UTF-8. Non-UTF-8 samples are now judged by their control bytes. +- On PHP 8.5 every object walked raised three `SplObjectStorage` deprecations, + which Laravel logs - and a log record raised from inside a log tap is redacted, + which raises them again. Active objects are now tracked by `spl_object_id`. +- `operators.default` had no effect on anything found by a pattern. A rule can + always produce an operator from its `mode`, which defaults to replace, and + that default was treated as a choice - so it outranked the profile default + and made the setting silently unreachable. Only a rule that actually + configured an operator or a non-default mode now outranks it. +- A path pattern that is purely numeric - `'0' => 'redact'`, or `items.0` + written as a key - crashed. PHP turns a numeric array key into an integer, + which reached a parameter typed as string. + +- Recursion is depth-bounded and cycle-aware. A self-referencing `toArray()` + used to exhaust memory and kill the process. (R-03) +- The logging path never throws. A bad profile no longer takes the channel down, + and diagnostics cannot re-enter the logger that raised them. (R-04) +- Scanner exclude patterns work. `vendor/*` and `node_modules/*` were passed to + `Finder::notName()`, which matches basenames, so they matched nothing and + every dependency was scanned. Binary files and gitignored files are skipped + too. (R-05) +- Redaction metadata no longer corrupts the payload: a list stays a list, and a + caller's own `_redacted` key is not overwritten. Prefer + `inspect()`. (R-07) +- `safe_keys` supports the wildcards the README has always documented. (R-08) +- Documented environment variables take effect. `REDACTOR_MAX_OBJECT_SIZE` was + silently ignored and `REDACTOR_SCAN_MAX_FILE_SIZE` crashed the scan + command. (R-09) +- PCRE failures fail closed. `preg_match()` returning `false` was read as + "no match", so an errored pattern let the value through. (R-15) +- Entropy is measured per character, not per byte, and can be judged per + alphabet. The `aws_secret_key` pattern no longer matches any 40-character + alphanumeric run. (R-16) +- Checksum validators (`luhn`, `iban`, `ssn`) reject values of the right shape + that cannot be the real thing. (R-17) +- `mergeConfigFrom()` runs in `register()`, not `boot()`. (R-14) +- `RedactorFormatter::formatBatch()` formats every record; it used to return only + the first, so batching handlers dropped the rest. (R-06) + +### Packaging and CI + +- Line coverage is 100% and the CI floor is set there. Dead code the detection + seam had left behind is gone, including `Operator::isPreserving()`, which + nothing read: whether an operator preserved a value is derived from its + output. +- Documentation moved from the README into `docs/`: getting started, + configuration reference, rules, operators and pseudonymisation, boundaries, + scanning, entity recognition, testing, extending and upgrading from 0.1.0. + The README is an overview and index. +- Rector with the PHP 8.3, dead-code, code-quality, type-declaration, + early-return and Laravel sets, applied to the tree and enforced by the + pre-commit hook, `composer preflight` and the static-analysis workflow. +- A security workflow: Semgrep on the PHP and secrets rulesets, and + `composer audit`, on push, on pull requests and weekly. + +- PHP 8.5 supported and in the test matrix. (R-21) +- `Tests\` no longer ships in the production autoload; `.gitattributes` keeps + development files out of the dist archive. (R-18) +- Dropped the unused `spatie/laravel-package-tools` requirement and declared + `symfony/finder` and `monolog/monolog`, which the package uses directly. (R-19) +- Coverage floor of 90% (CI reports 95.3%), a performance regression suite run + without coverage instrumentation, and + `failOnWarning`/`failOnRisky`/`failOnDeprecation` in phpunit.xml. (R-24) +- Mutation testing (Pest's built-in mutator) is available locally via + `composer mutate`. It is not run in CI: two consecutive runs over an + identical 1,557-mutant set scored 70.6% and 66.7% with only added passing + tests between them, so the number is not stable enough to act on + automatically. (R-24) +- Boundary tests for every redaction threshold - max_object_size, + max_value_length, the entropy threshold and min_length, max_depth, partial + mode's `keep`, and the Luhn length window. Mutation testing surfaced these: + the thresholds were covered but never their edges, so `>` could become `>=` + without a test noticing. On a redactor an off-by-one there is the difference + between catching a secret and emitting it. +- **Laravel 11 support dropped**; the package now requires + `illuminate/support ^12.0|^13.0`. Every 11.x release is flagged by a + Packagist security advisory, so Composer's default policy refuses to install + any of them, making the declared support unusable in practice. +- **Laravel 13 supported** and in the test matrix. `symfony/finder` widened to + `^7.0|^8.0`, which Laravel 13 requires. +- Dropped the unused `pestphp/pest-plugin-laravel` dev dependency. It was the + only thing pinning the test toolchain to a single Laravel major, and no test + used it - `$this->artisan()` comes from Testbench. +- The test matrix no longer uses `fail-fast`. +- Pint passes. `LICENCE.md` renamed to `LICENSE.md` so the README links and the + Packagist licence detection work. (R-22, R-23) + ## v0.1.0 - 2025-06-19 Redactor v0.1.0 diff --git a/LICENCE.md b/LICENSE.md similarity index 100% rename from LICENCE.md rename to LICENSE.md diff --git a/README.md b/README.md index 0236c60..718f548 100644 --- a/README.md +++ b/README.md @@ -1,531 +1,147 @@ # Kirschbaum Redactor -![Laravel Supported Versions](https://img.shields.io/badge/laravel-10.x/11.x/12.x-green.svg) +![Laravel Supported Versions](https://img.shields.io/badge/laravel-12.x%20%7C%2013.x-green.svg) [![MIT Licensed](https://img.shields.io/badge/license-MIT-brightgreen.svg?style=flat-square)](LICENSE.md) [![Latest Version on Packagist](https://img.shields.io/packagist/v/kirschbaum-development/redactor.svg?style=flat-square)](https://packagist.org/packages/kirschbaum-development/redactor) ![Application Testing](https://github.com/kirschbaum-development/redactor/actions/workflows/php-tests.yml/badge.svg) ![Static Analysis](https://github.com/kirschbaum-development/redactor/actions/workflows/static-analysis.yml/badge.svg) ![Code Style](https://github.com/kirschbaum-development/redactor/actions/workflows/style-check.yml/badge.svg) -Automatically redact sensitive data from arrays, objects, and strings before logging or exporting. Features a class-based strategy system with profile-based configurations, Shannon entropy detection. - -> This package is in active development and its API can change abruptly without any notice. Please reach out if you plan to use it in a production environment. +Redactor removes sensitive data from anything a Laravel application emits before it leaves: log records, HTTP responses, streamed output, MCP tool results, prompts sent to a language model, exports and queued jobs. It finds sensitive values by the key they sit under, by what they look like (credential patterns with checksum validators, Shannon entropy, an optional named entity recogniser) and by where they live in a payload, then replaces only the sensitive span so the text around it survives. +What replaces a value is a separate, per-entity decision. The same email address can become `[REDACTED]` in an audit log, a stable pseudonym like `u_7f3ac9@customer.com` in an application log so counts and joins still work, or a reversible token like `tok_email_k4m9rp2xzq` in front of a model so the application can act on the answer. Profiles bundle the rules and the decisions, and every boundary takes a profile name. The same engine scans files and git history from `redactor:scan`, with SARIF output, baselines and optional live-credential verification. ## Quick Start +Install the package and publish its configuration: + ```bash composer require kirschbaum-development/redactor php artisan vendor:publish --tag=redactor-config ``` -The package automatically registers the service provider and facade. Use it directly: - -```php -use Kirschbaum\Redactor\Facades\Redactor; - -// Basic usage -$data = [ - 'user_id' => 123, - 'password' => 'secret123', - 'api_key' => 'sk-1234567890abcdef1234567890abcdef12345678', - 'email' => 'user@example.com' -]; - -$redacted = Redactor::redact($data); -// Result: -// [ -// 'user_id' => 123, // Safe key - preserved -// 'password' => '[REDACTED]', // Blocked key - redacted -// 'api_key' => '[REDACTED]', // High entropy - redacted -// 'email' => '[REDACTED]', // Email pattern - redacted -// '_redacted' => true // Metadata added -// ] -``` - -## Core Concepts - -### Redaction Strategies - -The package uses a class-based configuration: - -1. **SafeKeysStrategy** - Preserves safe keys like `id`, `user_id` -2. **BlockedKeysStrategy** - Always redacts blocked keys like `password`, `secret` -3. **LargeObjectStrategy** - Redacts objects/arrays exceeding size limits -4. **LargeStringStrategy** - Redacts strings exceeding length limits -5. **RegexPatternsStrategy** - Custom regex patterns for emails, credit cards, etc. -6. **ShannonEntropyStrategy** - Detects high-entropy strings (API keys, tokens) - -### Profiles - -Profiles provide different redaction configurations for different contexts: - -```php -// Use built-in profiles -$logData = Redactor::redact($data, 'default'); // Balanced redaction -$auditData = Redactor::redact($data, 'strict'); // Aggressive redaction -$debugData = Redactor::redact($data, 'performance'); // Minimal redaction for speed -``` - -## Configuration - -The config file (`config/redactor.php`) uses a class-based approach: - -```php -return [ - 'default_profile' => 'default', - - 'profiles' => [ - 'default' => [ - 'enabled' => true, - - // Strategies executed in array order (top-to-bottom priority) - 'strategies' => [ - \Kirschbaum\Redactor\Strategies\SafeKeysStrategy::class, - \Kirschbaum\Redactor\Strategies\BlockedKeysStrategy::class, - \Kirschbaum\Redactor\Strategies\LargeObjectStrategy::class, - \Kirschbaum\Redactor\Strategies\LargeStringStrategy::class, - \Kirschbaum\Redactor\Strategies\RegexPatternsStrategy::class, - \Kirschbaum\Redactor\Strategies\ShannonEntropyStrategy::class, - ], - - 'safe_keys' => ['id', 'user_id', 'uuid', 'created_at', 'updated_at'], - 'blocked_keys' => ['password', 'secret', 'token', 'api_key', 'authorization'], - 'patterns' => [ - 'email' => '/[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+/', - 'credit_card' => '/\b(?:\d[ -]*?){13,16}\b/', - 'ssn' => '/\b\d{3}-?\d{2}-?\d{4}\b/', - 'phone_simple' => '/\b\d{3}[.-]?\d{3}[.-]?\d{4}\b/', - 'url_with_auth' => '/https?:\/\/[^:\/\s]+:[^@\/\s]+@[^\s]+/', - ], - 'replacement' => '[REDACTED]', - 'mark_redacted' => true, - 'track_redacted_keys' => false, - 'non_redactable_object_behavior' => 'preserve', // 'preserve', 'remove', 'redact', 'empty_array' - 'max_value_length' => 5000, - 'redact_large_objects' => true, - 'max_object_size' => 100, - - 'shannon_entropy' => [ - 'enabled' => true, - 'threshold' => 4.8, // Higher = more selective - 'min_length' => 25, // Only analyze strings this long or longer - 'exclusion_patterns' => [ - '/^https?:\/\//', // URLs - '/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i', // UUIDs - '/^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$/', // IP addresses - '/^[0-9a-f]{2}:[0-9a-f]{2}:[0-9a-f]{2}:[0-9a-f]{2}:[0-9a-f]{2}:[0-9a-f]{2}$/i', // MAC addresses - ], - ], - ], - ], -]; -``` - -## Wildcard Patterns - -The `BlockedKeysStrategy` and `SafeKeysStrategy` support powerful wildcard patterns using the `*` character. This allows you to match multiple key variations without listing each one explicitly. - -### Basic Wildcard Usage +Add the tap to a log channel in `config/logging.php`. It pushes a Monolog processor, so the channel keeps its own formatter: ```php -// config/redactor.php -'profiles' => [ - 'wildcard_example' => [ - 'enabled' => true, - 'strategies' => [ - \Kirschbaum\Redactor\Strategies\BlockedKeysStrategy::class, - ], - 'blocked_keys' => [ - '*token*', // Matches any key containing "token" - '*key*', // Matches any key containing "key" - 'password', // Exact match (no wildcards) - 'user_*_data', // Matches keys like "user_profile_data", "user_settings_data" - ], - // ... other config - ], -]; - -// Usage example -$data = [ - 'user_id' => 123, - 'api_token' => 'secret123', // Matched by *token* - 'access_token' => 'abc123', // Matched by *token* - 'my_custom_token' => 'xyz789', // Matched by *token* - 'user_api_key' => 'key123', // Matched by *key* - 'private_key_data' => 'private', // Matched by *key* - 'password' => 'secret', // Matched by exact "password" - 'user_profile_data' => 'profile', // Matched by user_*_data - 'user_settings_data' => 'settings', // Matched by user_*_data - 'normal_field' => 'safe_value', // Not matched - preserved -]; - -$redacted = Redactor::redact($data, 'wildcard_example'); -``` - -### Wildcard Pattern Types - -#### Contains Pattern (`*word*`) -Matches any key that contains the specified word anywhere: - -```php -'blocked_keys' => ['*token*', '*secret*', '*auth*'], - -// Matches: -// - api_token, access_token, token_data, my_token_field -// - user_secret, secret_key, app_secret_config -// - auth_header, oauth_token, authentication_data -``` - -#### Prefix Pattern (`word*`) -Matches any key that starts with the specified word: - -```php -'blocked_keys' => ['password*', 'secret*', 'api*'], - -// Matches: -// - password, password_hash, password_confirmation -// - secret, secret_key, secret_data -// - api, api_key, api_token, api_endpoint -``` - -#### Suffix Pattern (`*word`) -Matches any key that ends with the specified word: - -```php -'blocked_keys' => ['*token', '*key', '*secret'], - -// Matches: -// - access_token, api_token, user_token -// - private_key, public_key, encryption_key -// - user_secret, app_secret, database_secret -``` - -#### Multi-Wildcard Patterns (`word*middle*word`) -Use multiple wildcards for complex patterns: - -```php -'blocked_keys' => [ - 'user_*_token', // user_api_token, user_auth_token - 'app_*_*_key', // app_private_encryption_key, app_public_signing_key - '*_key_*', // my_key_data, the_key_value, user_key_config +'stack' => [ + 'driver' => 'stack', + 'channels' => explode(',', env('LOG_STACK', 'single')), + 'tap' => [Kirschbaum\Redactor\Logging\RedactorTap::class], ], ``` -### Case-Insensitive Matching - -All wildcard patterns are case-insensitive by default: - -```php -'blocked_keys' => ['*TOKEN*'], - -// Matches all of these: -// - API_TOKEN, api_token, Api_Token, MyTokenData, user_token_field -``` - -### Combining Exact and Wildcard Patterns - -You can mix exact matches with wildcard patterns in the same configuration: - -```php -'blocked_keys' => [ - 'password', // Exact match - 'secret', // Exact match - '*token*', // Wildcard pattern - '*_key_*', // Complex wildcard - 'user_*_data', // Specific structure -], - -'safe_keys' => [ - 'id', // Exact match - always preserved - 'user_id', // Exact match - always preserved - '*_count', // Wildcard pattern - preserve counting fields - 'meta_*', // Wildcard pattern - preserve metadata fields -], -``` - -### Performance Considerations - -- Exact matches are faster than wildcard patterns -- Simple wildcards (`*word*`) are faster than complex multi-wildcard patterns -- Consider placing more specific patterns before broader ones -- Use exact matches when you know the specific key names - -## Common Use Cases - -### Logging Context +Redact anything else directly: ```php use Kirschbaum\Redactor\Facades\Redactor; -// Before logging user actions -Log::info('User action', Redactor::redact([ +Redactor::redact([ 'user_id' => 123, - 'action' => 'login', - 'ip_address' => '192.168.1.1', - 'session_token' => 'abc123def456...', - 'user_agent' => 'Mozilla/5.0...', - 'api_response' => $sensitiveApiData, -])); -``` - -### Laravel Logging Integration - -For automatic redaction of all log entries, use the `CustomLogTap` with Laravel's logging configuration. In your `config/logging.php`, add the tap to any channel: - -```php -'channels' => [ - 'stack' => [ - 'driver' => 'stack', - 'channels' => explode(',', env('LOG_STACK', 'single')), - 'ignore_exceptions' => false, - 'tap' => [Kirschbaum\Redactor\Logging\CustomLogTap::class], - ], - - 'single' => [ - 'driver' => 'single', - 'path' => storage_path('logs/laravel.log'), - 'level' => env('LOG_LEVEL', 'debug'), - 'tap' => [Kirschbaum\Redactor\Logging\CustomLogTap::class], - ], - - 'daily' => [ - 'driver' => 'daily', - 'path' => storage_path('logs/laravel.log'), - 'level' => env('LOG_LEVEL', 'debug'), - 'days' => 14, - 'tap' => [Kirschbaum\Redactor\Logging\CustomLogTap::class], - ], -], -``` - -With this configuration, all log entries will automatically have their context data redacted before being written to logs. The tap uses the `default` redaction profile unless otherwise configured. - -### API Response Sanitization - -```php -use Kirschbaum\Redactor\Facades\Redactor; - -// Before returning debug information -return response()->json([ - 'debug' => Redactor::redact($requestData, 'performance'), - 'status' => 'processed' + 'password' => 'hunter2', + 'email' => 'bob@example.com', + 'note' => 'card 4111 1111 1111 1111 on file', ]); -``` - -### Database Export & Auditing - -```php -use Kirschbaum\Redactor\Facades\Redactor; - -// Before exporting user data -$users = User::all()->map(function ($user) { - return Redactor::redact($user->toArray(), 'strict'); -}); - -// Audit trail with sensitive data redacted -$auditLog = Redactor::redact([ - 'user_id' => $user->id, - 'changes' => $changes, - 'request_data' => request()->all(), -], 'audit'); -``` - -### PCI Compliance Example -```php -// config/redactor.php -'profiles' => [ - 'pci_compliant' => [ - 'enabled' => true, - 'strategies' => [ - \Kirschbaum\Redactor\Strategies\SafeKeysStrategy::class, - \Kirschbaum\Redactor\Strategies\BlockedKeysStrategy::class, - \Kirschbaum\Redactor\Strategies\RegexPatternsStrategy::class, - ], - 'safe_keys' => ['order_id', 'customer_id', 'amount', 'currency'], - 'blocked_keys' => [ - 'credit_card', 'cc_number', 'card_number', 'pan', - 'cvv', 'cvc', 'cvn', 'expiry', 'exp_date', 'security_code' - ], - 'patterns' => [ - 'credit_card' => '/\b(?:\d[ -]*?){13,16}\b/', - 'ssn' => '/\b\d{3}-?\d{2}-?\d{4}\b/', - 'routing_number' => '/\b\d{9}\b/', - ], - 'replacement' => '[PCI_REDACTED]', - 'non_redactable_object_behavior' => 'redact', - ], -]; - -// Usage -$orderData = Redactor::redact($order->toArray(), 'pci_compliant'); +// [ +// 'user_id' => 123, +// 'password' => '[REDACTED]', +// 'email' => '[REDACTED]', +// 'note' => 'card ***************1111 on file', +// '_redacted' => true, +// ] ``` -## Advanced Features - -### Object Handling - -The package handles various object types: +Ask what was found rather than reading it back out of the payload: ```php -use Kirschbaum\Redactor\Facades\Redactor; - -// Laravel models (uses toArray()) -$user = User::find(1); -$redacted = Redactor::redact($user); +$result = Redactor::profile('strict')->withoutMarkers()->inspect($data); -// Plain objects (uses JSON serialization) -$object = new stdClass(); -$object->secret = 'sensitive'; -$redacted = Redactor::redact($object); - -// Non-serializable objects (configurable behavior) -$resource = fopen('file.txt', 'r'); -$redacted = Redactor::redact(['file' => $resource]); -// Behavior controlled by 'non_redactable_object_behavior' setting +$result->value; // the redacted payload +$result->wasRedacted; // true +$result->redactedKeys; // ['password', 'email'] +$result->findings; // rule, entity, key, offset, length and confidence for each match ``` -### Custom Strategies - -Create your own redaction logic with full type safety: - -```php -use Kirschbaum\Redactor\Strategies\RedactionStrategyInterface; -use Kirschbaum\Redactor\RedactionContext; - -class InternalDataStrategy implements RedactionStrategyInterface -{ - public function shouldHandle(mixed $value, string $key, RedactionContext $context): bool - { - return str_contains($key, 'internal_') || str_contains($key, 'debug_'); - } - - public function handle(mixed $value, string $key, RedactionContext $context): mixed - { - $context->markRedacted(); - return '[INTERNAL]'; - } -} - -// Register and use -use Kirschbaum\Redactor\Facades\Redactor; +## How It Works -Redactor::registerCustomStrategy('internal_data', new InternalDataStrategy()); +A **profile** is one complete configuration: the strategies to run, the keys that are safe or blocked, the patterns to look for, and what to do with what is found. Five ship: `default`, `strict`, `observability`, `file_scan` and `performance`. Every entry point takes a profile name, so the same value can be pseudonymised on one channel and removed on another. -// Add to profile configuration -'strategies' => [ - 'internal_data', // Custom strategy by registered name - \Kirschbaum\Redactor\Strategies\SafeKeysStrategy::class, - // ... other strategies -], -``` +**Strategies** run in the order the profile lists them. Key rules decide by the name a value sits under; pattern rules, known secrets, entropy and entity recognition decide by content and report what they found and where; path rules decide by location and are checked before anything else. Detections are resolved once, so two rules matching the same text produce one rewrite and a surrogate written for one detection is never re-detected by the next. -### Multiple Usage Patterns +**Operators** decide what replaces a detection, per entity rather than per rule: ```php -// Via Facade (recommended) -use Kirschbaum\Redactor\Facades\Redactor; -$result = Redactor::redact($data, 'profile_name'); - -// Via Service Container -$redactor = app(\Kirschbaum\Redactor\Redactor::class); -$result = $redactor->redact($data, 'profile_name'); - -// Direct Instantiation (gets fresh instance - no state conflicts) -$redactor = new \Kirschbaum\Redactor\Redactor(); -$result = $redactor->redact($data, 'profile_name'); - -// Check available profiles -$profiles = Redactor::getAvailableProfiles(); -$exists = Redactor::profileExists('custom_profile'); -``` - -## Built-in Profiles - -- **`default`**: Balanced redaction for general logging and debugging -- **`strict`**: Aggressive redaction for sensitive contexts and audit trails -- **`performance`**: Minimal redaction optimized for high-throughput scenarios - -## Environment Configuration - -Many settings can be controlled via environment variables: - -```env -REDACTOR_ENABLED=true -REDACTOR_DEFAULT_PROFILE=default -REDACTOR_REPLACEMENT="[REDACTED]" -REDACTOR_MARK_REDACTED=true -REDACTOR_TRACK_KEYS=false -REDACTOR_OBJECT_BEHAVIOR=preserve -REDACTOR_MAX_VALUE_LENGTH=5000 -REDACTOR_LARGE_OBJECTS=true -REDACTOR_MAX_OBJECT_SIZE=100 -REDACTOR_SHANNON_ENABLED=true -REDACTOR_SHANNON_THRESHOLD=4.8 -REDACTOR_SHANNON_MIN_LENGTH=25 +'operators' => [ + 'default' => 'redact', // [REDACTED] + 'email' => ['surrogate' => ['preserve_domain' => true]], // u_7f3ac9@customer.com, stable + 'credit_card' => ['partial' => ['keep' => 4]], // ************1111 + 'ssn' => 'nullify', // null, so a typed field stays typed +], ``` -## File Scanning Command +`surrogate`, `hash` and `tokenize` are keyed with an HMAC derived from `APP_KEY` (or a key of your own), so the same input always yields the same stand-in and logs stay joinable without a route back to the original. -The package includes a console command to scan files and directories for sensitive content: +**Scanning** runs the same rules over files and git history: ```bash -# Scan specific files -php artisan redactor:scan path/to/sensitive-file.txt - -# Scan directories (scans entire project by default) -php artisan redactor:scan app/ config/ - -# Scan with custom profile -php artisan redactor:scan --profile=strict app/ - -# Exit with error code if sensitive content found (useful for CI) -php artisan redactor:scan --bail app/ - -# JSON output for programmatic use -php artisan redactor:scan --output=json config/ - -# Summary only (no per-file details) -php artisan redactor:scan --summary-only -``` - -The scanner uses the `file_scan` profile by default, which is optimized for plain text content and detects: -- API keys, tokens, and secrets -- Email addresses and personal information -- High-entropy strings (potential keys/tokens) -- Credit cards, SSNs, phone numbers -- Passwords and authentication strings - -Results show **CLEAN**, **FINDINGS**, or **SKIPPED** status for each file, with a summary of total files scanned and findings detected. +php artisan redactor:scan --staged --bail # the pre-commit gate +php artisan redactor:scan --diff=origin/main --output=sarif > redactor.sarif +php artisan redactor:scan --update-baseline # accept what is already there +``` + +## What It Covers + +- **Log channels** through `RedactorTap`, which never throws: a broken profile replaces the record rather than taking the channel down. +- **HTTP responses** through the `redact` middleware: JSON as data, text as text, streams as they stream, files untouched, failing closed to a 500. +- **Streams** through `StreamRedactor`, which holds back a window so a secret split across two chunks is still caught. +- **MCP servers** through the `RedactsResponses` trait on a Laravel MCP server: tool results, structured content, resources, prompts, streamed output and errors. +- **AI agents** through the `RedactPrompt` middleware for Laravel's AI package: the prompt is redacted on the way out and tokens are resolved in the answer. +- **Exports, jobs, error reporters and third-party clients** through `Redactor::redact()` and `redactSafely()` with a profile per destination. +- **Files and git history** through `redactor:scan`, with table, JSON, SARIF and JUnit output, `--staged`, `--diff` and `--history` modes, baselines, inline `redactor:allow` markers, and a publishable pre-commit hook and GitHub workflow. +- **Your test suite** through `Redactor::fake()`, so a test can assert that a secret never left. +- **Names, places and organisations in prose** through a Presidio-compatible recogniser, or in-process with no sidecar through the companion package [`kirschbaum-development/redactor-onnx`](https://github.com/kirschbaum-development/redactor-onnx). + +## Documentation + +The full documentation lives in [`docs/`](docs/README.md): + +| Page | What it covers | +| --- | --- | +| [Getting Started](docs/getting-started.md) | Installation, the Monolog tap, `redact()`, `inspect()`, the fluent builder, profiles and the five that ship. | +| [Configuration](docs/configuration.md) | Every key in `config/redactor.php` with its type, default and environment variable; the shipped profiles compared. | +| [Rules](docs/rules.md) | Pattern rules, validators, keywords, samples, dictionary rules, allow-lists, path rules, safe and blocked keys, known secrets, confidence and how detections are resolved. | +| [Operators and Pseudonymisation](docs/operators-and-pseudonymisation.md) | Every operator with example output, precedence, surrogates, the key and salt, reversible tokens and `detokenize()`. | +| [Boundaries](docs/boundaries.md) | Log channels, HTTP responses, streams, MCP servers, AI agents, exports and jobs, and the `RedactionPerformed` event. | +| [Scanning](docs/scanning.md) | `redactor:scan` in full: paths, output formats, git modes, decoding, baselines, suppression, verification, the hook and workflow, exit codes. | +| [Entity Recognition](docs/entity-recognition.md) | Finding names, places and organisations in prose with a Presidio-compatible recogniser or the in-process `redactor-onnx` package, and when not to. | +| [Testing](docs/testing.md) | `Redactor::fake()` and its assertions, `redactor:validate`, rule samples, the package's own test conventions. | +| [Extending](docs/extending.md) | Every contract, how to register each, a worked custom strategy and operator, and macros. | +| [Upgrading](docs/upgrading.md) | Every renamed class and method from 0.1.0, every behaviour change, and what to do about each. | ## Requirements -- PHP 8.3+ -- Laravel 11.x or 12.x +- PHP 8.3, 8.4 or 8.5 +- Laravel 12.x or 13.x -## Installation - -```bash -composer require kirschbaum-development/redactor -php artisan vendor:publish --tag=redactor-config -``` +`laravel/mcp` and `laravel/ai` are suggested, not required; the adapters for them are only loaded when you use them. ## Testing ```bash -# Run tests -./vendor/bin/pest - -# Run tests with coverage -./vendor/bin/pest --coverage +composer test # full suite, in parallel +composer test-coverage # with the coverage floor enforced +composer lint # Pint, Rector, PHPStan (level 10, no baseline) +composer rector:check # what Rector would change, without changing it +composer mutate # mutation testing (Pest); local only, not run in CI +composer preflight # everything CI runs ``` -## Roadmap -- Add Laravel custom log formatter to tap logs and automatically redact sensitive data -- Add supoprt for partial replacement of sensitive data (low priority) +Coverage and mutation testing need a coverage driver (pcov or Xdebug) loaded in the CLI. See [Testing](docs/testing.md) for the conventions and for `Redactor::fake()` in your own suite. + +## Changelog + +See [CHANGELOG.md](CHANGELOG.md) for what changed in each release and [Upgrading](docs/upgrading.md) for how to move from 0.1.0 to 1.0.0. ## License MIT License. See [LICENSE.md](LICENSE.md) for details. - diff --git a/composer.json b/composer.json index f4ec934..b4b29cb 100644 --- a/composer.json +++ b/composer.json @@ -21,8 +21,7 @@ ], "autoload": { "psr-4": { - "Kirschbaum\\Redactor\\": "src/", - "Tests\\": "tests/" + "Kirschbaum\\Redactor\\": "src/" } }, "autoload-dev": { @@ -40,12 +39,15 @@ } ], "require-dev": { - "pestphp/pest": "^3.8", - "laravel/pint": "^1.22", "larastan/larastan": "^3.4", - "orchestra/testbench": "^10.3", - "pestphp/pest-plugin-laravel": "^3.1", - "timacdonald/log-fake": "^2.4" + "laravel/pint": "^1.22", + "orchestra/testbench": "^10.3|^11.0", + "pestphp/pest": "^3.8", + "timacdonald/log-fake": "^2.4", + "laravel/mcp": "^1.0@beta", + "laravel/ai": "^0.11", + "rector/rector": "^2.6", + "driftingly/rector-laravel": "^2.5" }, "config": { "allow-plugins": { @@ -53,9 +55,11 @@ } }, "require": { - "illuminate/support": "^11.9|^12.0", - "spatie/laravel-package-tools": "^1.16", - "php": "^8.3|^8.4" + "php": "^8.3|^8.4|^8.5", + "illuminate/support": "^12.0|^13.0", + "monolog/monolog": "^3.0", + "symfony/finder": "^7.0|^8.0", + "symfony/process": "^7.0|^8.0" }, "extra": { "laravel": { @@ -85,7 +89,8 @@ ], "lint": [ "@php vendor/bin/pint --ansi", - "@php vendor/bin/phpstan analyse --verbose --ansi" + "@php vendor/bin/rector process --ansi", + "@php vendor/bin/phpstan analyse --verbose --ansi --memory-limit=1G" ], "test": [ "@clear", @@ -93,8 +98,26 @@ ], "preflight": [ "@php vendor/bin/pint --test --ansi", - "@php vendor/bin/phpstan analyse --no-progress --ansi", + "@php vendor/bin/rector process --dry-run --ansi", + "@php vendor/bin/phpstan analyse --no-progress --ansi --memory-limit=1G", "@php vendor/bin/pest --parallel" + ], + "test-coverage": [ + "@clear", + "@php -d memory_limit=2G vendor/bin/pest --coverage --min=100" + ], + "mutate": [ + "@php -d memory_limit=2G vendor/bin/pest --mutate --everything --covered-only" + ], + "rector": [ + "@php vendor/bin/rector process --ansi" + ], + "rector:check": [ + "@php vendor/bin/rector process --dry-run --ansi" ] + }, + "suggest": { + "laravel/mcp": "To redact everything an MCP server returns with the RedactsResponses trait", + "laravel/ai": "To redact prompts and resolve tokens in answers with the RedactPrompt middleware" } -} \ No newline at end of file +} diff --git a/config/redactor.php b/config/redactor.php index 6985e3a..6ede9dd 100644 --- a/config/redactor.php +++ b/config/redactor.php @@ -1,6 +1,239 @@ [ + // Any scheme - postgres://, redis://, amqp://, https:// - and only the + // password is replaced, so the host and path stay readable. + 'pattern' => '/([a-z][a-z0-9+.-]*:\/\/[^:\/\s@]*:)([^@\/\s]+)(@)/i', + 'capture' => 2, + 'entity' => 'url_credentials', + 'samples' => ['https://admin:hunter2@db.example.com/x', 'postgres://app:s3cr3t@db.internal:5432/app'], + 'counter_samples' => ['https://db.example.com/x', 'user@example.com'], + 'min_length' => 7, + 'confidence' => 0.9, + 'keywords' => ['://'], + ], + 'private_key_block' => [ + 'pattern' => '/-----BEGIN (?:[A-Z ]+ )?PRIVATE KEY-----[\s\S]*?-----END (?:[A-Z ]+ )?PRIVATE KEY-----/', + 'entity' => 'private_key', + 'samples' => ["-----BEGIN RSA PRIVATE KEY-----\nMIIEow\n-----END RSA PRIVATE KEY-----"], + 'counter_samples' => ['-----BEGIN CERTIFICATE-----'], + 'min_length' => 52, + 'confidence' => 1.0, + 'keywords' => ['private key'], + ], + 'jwt' => [ + 'pattern' => '/\beyJ[A-Za-z0-9_-]{5,}\.eyJ[A-Za-z0-9_-]{5,}\.[A-Za-z0-9_-]{5,}\b/', + 'entity' => 'jwt', + 'samples' => ['eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c'], + 'counter_samples' => ['eyJ.not.a.jwt'], + 'min_length' => 23, + 'confidence' => 0.9, + 'keywords' => ['eyj'], + ], + 'bearer_token' => [ + 'pattern' => '/(bearer\s+)([A-Za-z0-9._~+\/=-]{16,})/i', + 'capture' => 2, + 'entity' => 'bearer_token', + 'samples' => ['Authorization: Bearer 8f14e45fceea167a5a36dedd4bea2543'], + 'counter_samples' => ['Bearer of bad news'], + 'min_length' => 23, + 'confidence' => 0.85, + 'keywords' => ['bearer'], + ], + 'aws_access_key' => [ + 'pattern' => '/\b(?:AKIA|ASIA)[0-9A-Z]{16}\b/', + 'entity' => 'aws_access_key', + 'samples' => ['AKIAIOSFODNN7EXAMPLE', 'ASIAIOSFODNN7EXAMPLE'], + 'counter_samples' => ['AKIA_NOT_A_KEY'], + 'min_length' => 20, + 'confidence' => 0.9, + 'keywords' => ['akia', 'asia'], + ], + 'github_token' => [ + 'pattern' => '/\b(?:gh[pousr]_[A-Za-z0-9]{36,255}|github_pat_[A-Za-z0-9_]{22,255})\b/', + 'entity' => 'github_token', + 'samples' => ['ghp_16C7e42F292c6912E7710c838347Ae178B4a', 'github_pat_11ABCDEFG0123456789_abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ0123'], + 'counter_samples' => ['ghp_short'], + 'min_length' => 33, + 'confidence' => 0.95, + 'keywords' => ['ghp_', 'gho_', 'ghu_', 'ghs_', 'ghr_', 'github_pat_'], + ], + 'stripe_key' => [ + // Secret and restricted keys only; publishable keys are meant to be seen. + 'pattern' => '/\b(?:sk|rk)_(?:live|test)_[A-Za-z0-9]{10,99}\b/', + 'entity' => 'stripe_key', + 'samples' => ['sk_live_4eC39HqLyjWDarjtT1zdp7dc', 'rk_test_4eC39HqLyjWDarjtT1zdp7dc'], + 'counter_samples' => ['pk_live_4eC39HqLyjWDarjtT1zdp7dc'], + 'min_length' => 18, + 'confidence' => 0.95, + 'keywords' => ['sk_', 'rk_'], + ], + 'slack_token' => [ + 'pattern' => '/\bxox[abpors]-[A-Za-z0-9-]{10,}\b/', + 'entity' => 'slack_token', + 'samples' => ['xoxb-1234567890-abcdefghijABCDEFGHIJ'], + 'counter_samples' => ['xoxz-not-a-token'], + 'min_length' => 15, + 'confidence' => 0.9, + 'keywords' => ['xox'], + ], + 'anthropic_key' => [ + 'pattern' => '/\bsk-ant-[A-Za-z0-9_-]{20,}\b/', + 'entity' => 'anthropic_key', + 'samples' => ['sk-ant-api03-abcdefghijklmnopqrstuvwxyz'], + 'counter_samples' => ['sk-ant-short'], + 'min_length' => 27, + 'confidence' => 0.95, + 'keywords' => ['sk-ant-'], + ], + 'openai_key' => [ + 'pattern' => '/\bsk-(?:proj-)?[A-Za-z0-9_-]{20,}\b/', + 'entity' => 'openai_key', + 'samples' => ['sk-proj-abcdefghijklmnopqrstuvwxyz0123'], + 'counter_samples' => ['sk-short'], + 'min_length' => 23, + 'confidence' => 0.9, + 'keywords' => ['sk-'], + ], + 'google_api_key' => [ + 'pattern' => '/\bAIza[0-9A-Za-z_-]{35}\b/', + 'entity' => 'google_api_key', + 'samples' => ['AIzaSyA1234567890abcdefghijklmnopqrstuv'], + 'counter_samples' => ['AIza-not-a-key'], + 'min_length' => 39, + 'confidence' => 0.9, + 'keywords' => ['aiza'], + ], + 'sendgrid_key' => [ + 'pattern' => '/\bSG\.[A-Za-z0-9_-]{22}\.[A-Za-z0-9_-]{43}\b/', + 'entity' => 'sendgrid_key', + 'samples' => ['SG.abcdefghijklmnopqrstuv.abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQ'], + 'counter_samples' => ['SG.short.key'], + 'min_length' => 69, + 'confidence' => 0.95, + 'keywords' => ['sg.'], + ], +]; + +$identityPatterns = [ + 'email' => [ + // Byte-level rather than /u so a non-ASCII local part or domain + // matches without PCRE validating the whole value as UTF-8 first. + 'pattern' => '/[A-Za-z0-9_.+\-\x80-\xff]+@[A-Za-z0-9\-\x80-\xff]+(?:\.[A-Za-z0-9\-\x80-\xff]+)+/', + 'entity' => 'email', + 'confidence' => 0.8, + 'samples' => ['bob@example.com', 'josé@münchen.de'], + 'counter_samples' => ['not an email', 'user@localhost'], + 'min_length' => 5, + 'keywords' => ['@'], + ], + 'phone_formatted' => [ + // Needs separators or parentheses, so a date, a version or a card + // number is not mistaken for a phone number. + 'pattern' => '/(? 'phone', + 'confidence' => 0.6, + 'samples' => ['+44 20 7946 0958', '+1 (555) 867-5309', '555-867-5309'], + 'counter_samples' => ['2026-09-13 10:00:00', '4111 1111 1111 1111', 'v10.2.100'], + 'min_length' => 10, + ], + 'phone_e164' => [ + 'pattern' => '/(? 'phone', + 'confidence' => 0.7, + 'samples' => ['+447946095800'], + 'counter_samples' => ['+1'], + 'min_length' => 10, + 'keywords' => ['+'], + ], + 'phone_bare' => [ + // A bare ten-digit run is a Unix timestamp or an order number far + // more often than a phone number, so it needs a label nearby. + 'pattern' => '/(? 'phone', + 'confidence' => 0.5, + 'samples' => ['Phone: 5558675309'], + 'counter_samples' => ['started at 1694600000'], + 'min_length' => 10, + 'keywords' => ['phone', 'tel', 'mobile', 'cell', 'fax'], + ], + 'ssn' => [ + 'pattern' => '/\b\d{3}-\d{2}-\d{4}\b/', + // Rejects the never-issued area/group/serial values, which is most + // of what matches this shape by accident. + 'validator' => 'ssn', + 'entity' => 'ssn', + 'confidence' => 0.7, + 'samples' => ['ssn 123-45-6789'], + 'counter_samples' => ['000-12-3456', '666-12-3456'], + 'min_length' => 11, + ], + 'ssn_bare' => [ + 'pattern' => '/(? 'ssn', + 'entity' => 'ssn', + 'confidence' => 0.4, + 'samples' => ['SSN 123456789'], + 'counter_samples' => ['id 123456789'], + 'min_length' => 9, + 'keywords' => ['ssn', 'social security', 'tax id', 'tin'], + ], + 'credit_card' => [ + 'pattern' => '/\b(?:\d[ -]*?){13,16}\b/', + // Without the Luhn check this matches any 13-16 digit run: order + // numbers, tracking codes, concatenated timestamps. + 'validator' => 'luhn', + 'entity' => 'credit_card', + 'samples' => ['4111111111111111', '4111 1111 1111 1111'], + 'counter_samples' => ['1234567890123456'], + 'min_length' => 13, + ], + 'iban' => [ + // Accepts the spaced form banks print as well as the compact one. + 'pattern' => '/\b[A-Z]{2}\d{2}(?:[ ]?[A-Z0-9]{4}){2,7}(?:[ ]?[A-Z0-9]{1,4})?\b/', + 'validator' => 'iban', + 'entity' => 'iban', + 'samples' => ['DE89 3704 0044 0532 0130 00', 'GB82WEST12345698765432'], + 'counter_samples' => ['DE00 0000 0000 0000 0000 00'], + 'min_length' => 12, + ], +]; return [ /* @@ -17,15 +250,405 @@ 'scan' => [ 'profile' => env('REDACTOR_SCAN_PROFILE', 'file_scan'), + + /* + | Glob patterns, matched against both the file's basename and its path + | relative to each scanned directory. A pattern ending in '/*' also + | prunes that directory during the walk rather than filtering its + | files one at a time. + */ 'exclude_patterns' => [ '*.lock', '*.min.js', + '*.map', 'vendor/*', 'node_modules/*', + 'storage/framework/*', + 'public/build/*', ], + 'max_file_size' => env('REDACTOR_SCAN_MAX_FILE_SIZE', 10_485_760), + + // Skip images, archives and compiled artefacts: scanning them + // produces nothing but entropy false positives. + 'skip_binary' => env('REDACTOR_SCAN_SKIP_BINARY', true), + + // Skip anything git is already ignoring. + 'respect_gitignore' => env('REDACTOR_SCAN_RESPECT_GITIGNORE', true), + + /* + | Files are scanned a window of lines at a time, so memory stays flat + | whatever the file size - the files most worth scanning are the large + | ones. Windows overlap so a secret spanning a boundary (a PEM block, a + | wrapped connection string) is still found; duplicates from the overlap + | are dropped by fingerprint. + */ + 'window_lines' => env('REDACTOR_SCAN_WINDOW_LINES', 512), + 'overlap_lines' => env('REDACTOR_SCAN_OVERLAP_LINES', 4), + + /* + | Look inside base64, URL-encoded and JSON-escaped spans, one layer + | deep. A credential URL in a JSON file reads `https:\/\/user:pass@`, + | which no plain pattern matches; a key in a Kubernetes secret is + | base64. Findings say which encoding hid them. + */ + 'decode' => env('REDACTOR_SCAN_DECODE', true), + + /* + |---------------------------------------------------------------------- + | Credential verification + |---------------------------------------------------------------------- + | + | Asks each provider whether a detected credential is still live, which + | turns a wall of maybes into a short list of keys to rotate today. + | + | It also sends real secrets to third parties. Nothing here happens + | unless all three of these agree: + | + | 1. enabled is true (this file, reviewable in a diff) + | 2. the run passes --verify (a human, per run) + | 3. the provider is listed below (who you are willing to tell) + | + | An empty list means none. Enabling the feature and choosing who to + | trust with the secrets are deliberately separate decisions, and + | redaction itself can never trigger this - only the scan command can. + | + */ + 'verification' => [ + 'enabled' => env('REDACTOR_SCAN_VERIFY', false), + + 'verifiers' => [ + // 'github_token', + // 'stripe_key', + // 'slack_token', + // 'openai_key', + // 'anthropic_key', + // 'sendgrid_key', + // 'google_api_key', + ], + ], + + /* + | Accepted findings, so CI fails on new secrets rather than on known + | ones. Generate with: + | + | php artisan redactor:scan --update-baseline + | + | The file stores hashed fingerprints, never the secrets themselves. + */ + 'baseline' => env('REDACTOR_SCAN_BASELINE', base_path('.redactor-baseline.json')), + ], + + /* + |-------------------------------------------------------------------------- + | Pseudonymization + |-------------------------------------------------------------------------- + | + | The `hash` and `surrogate` operators replace a value with a stable + | stand-in, so redacted logs stay joinable: the same email always produces + | the same surrogate, and you can still count distinct users or follow one + | account through a trace. + | + | The mapping is one-way (HMAC, not encryption). Anyone holding the key can + | confirm a guess, so the key must not travel with the logs. Leave it null + | to derive one from APP_KEY, which is never used directly. + | + | Rotating the key changes every surrogate. That is the intended way to + | break correlation with previously exported logs - and the reason not to + | rotate it casually. + | + */ + + 'pseudonymization' => [ + 'enabled' => env('REDACTOR_PSEUDONYMIZATION', true), + 'key' => env('REDACTOR_PSEUDONYMIZATION_KEY'), + + /* + | Mixed into every surrogate. Shared by every profile, so the same + | user gets the same surrogate on every channel and the logs stay + | joinable across them. A profile may set its own `pseudonymization` + | `salt` to deliberately break that correlation - an export that must + | not be linkable back to the application logs, say. + */ + 'salt' => env('REDACTOR_PSEUDONYMIZATION_SALT'), ], + /* + |-------------------------------------------------------------------------- + | Region Packs + |-------------------------------------------------------------------------- + | + | National identifiers and VAT numbers, grouped by country and switched on + | per profile with its `regions` list. Each rule carries a checksum + | validator where the identifier has one, keywords where the shape alone + | is too common (a nine-digit run is a phone number more often than a + | citizen number), and samples that redactor:validate proves. + | + | 'profiles' => ['default' => ['regions' => ['gb', 'nl', 'eu']]], + | + */ + + 'regions' => [ + 'gb' => [ + 'uk_national_insurance' => [ + 'pattern' => '/\b(?!BG|GB|NK|KN|TN|NT|ZZ)[A-CEGHJ-PR-TW-Z][A-CEGHJ-NPR-TW-Z] ?\d{2} ?\d{2} ?\d{2} ?[A-D]\b/', + 'entity' => 'national_id', + 'confidence' => 0.75, + 'min_length' => 9, + 'samples' => ['NI number AB 12 34 56 C', 'AB123456C'], + 'counter_samples' => ['AB12345C', 'ZZ123456C'], + ], + 'uk_nhs_number' => [ + 'pattern' => '/\b\d{3} ?\d{3} ?\d{4}\b/', + 'validator' => 'nhs', + 'keywords' => ['nhs'], + 'entity' => 'health_id', + 'confidence' => 0.7, + 'min_length' => 10, + 'samples' => ['NHS number 915 229 6008'], + 'counter_samples' => ['NHS number 123 456 7890', 'called 9152296008'], + ], + 'uk_vat' => [ + 'pattern' => '/\bGB ?\d{3} ?\d{4} ?\d{2}(?: ?\d{3})?\b/', + 'validator' => 'vat', + 'entity' => 'vat_number', + 'confidence' => 0.7, + 'min_length' => 11, + 'samples' => ['VAT GB436083107', 'GB 695 0749 92'], + 'counter_samples' => ['GB123456789'], + ], + ], + + 'nl' => [ + 'nl_bsn' => [ + 'pattern' => '/\b\d{9}\b/', + 'validator' => 'bsn', + 'keywords' => ['bsn', 'burgerservicenummer', 'sofinummer'], + 'entity' => 'national_id', + 'confidence' => 0.7, + 'min_length' => 9, + 'samples' => ['BSN 283194443'], + 'counter_samples' => ['BSN 123456789', 'order 283194443'], + ], + 'nl_vat' => [ + 'pattern' => '/\bNL ?\d{9} ?B ?\d{2}\b/', + 'validator' => 'vat', + 'entity' => 'vat_number', + 'confidence' => 0.7, + 'min_length' => 14, + 'samples' => ['NL514465888B07'], + 'counter_samples' => ['NL123456789B01'], + ], + ], + + 'de' => [ + 'de_steuer_id' => [ + 'pattern' => '/\b\d{2} ?\d{3} ?\d{3} ?\d{3}\b/', + 'validator' => 'steuer_id', + 'keywords' => ['steuer', 'idnr', 'tax', 'tin'], + 'entity' => 'national_id', + 'confidence' => 0.7, + 'min_length' => 11, + 'samples' => ['Steuer-ID 72 096 139 541'], + 'counter_samples' => ['Steuer-ID 12 345 678 901'], + ], + 'de_vat' => [ + 'pattern' => '/\bDE ?\d{9}\b/', + 'validator' => 'vat', + 'entity' => 'vat_number', + 'confidence' => 0.7, + 'min_length' => 11, + 'samples' => ['USt-IdNr. DE869428760'], + 'counter_samples' => ['DE000000000'], + ], + ], + + 'fr' => [ + 'fr_nir' => [ + 'pattern' => '/\b[12] ?\d{2} ?(?:0[1-9]|1[0-2]|[2-9]\d) ?(?:\d{2}|2A|2B) ?\d{3} ?\d{3} ?\d{2}\b/', + 'validator' => 'nir', + 'entity' => 'national_id', + 'confidence' => 0.8, + 'min_length' => 15, + 'samples' => ['NIR 1 96 04 20 350 020 61', '267127783624161'], + 'counter_samples' => ['1 96 04 20 350 020 62'], + ], + 'fr_vat' => [ + 'pattern' => '/\bFR ?[0-9A-Z]{2} ?\d{9}\b/', + 'validator' => 'vat', + 'entity' => 'vat_number', + 'confidence' => 0.7, + 'min_length' => 13, + 'samples' => ['TVA FR50786240626'], + 'counter_samples' => ['FR00786240626'], + ], + ], + + 'it' => [ + 'it_codice_fiscale' => [ + 'pattern' => '/\b[A-Z]{6}\d{2}[A-EHLMPRST]\d{2}[A-Z]\d{3}[A-Z]\b/i', + 'validator' => 'codice_fiscale', + 'entity' => 'national_id', + 'confidence' => 0.85, + 'min_length' => 16, + 'samples' => ['CF RSSMRA85T10A562S'], + 'counter_samples' => ['RSSMRA85T10A562T'], + ], + 'it_vat' => [ + 'pattern' => '/\bIT ?\d{11}\b/', + 'validator' => 'vat', + 'entity' => 'vat_number', + 'confidence' => 0.7, + 'min_length' => 13, + 'samples' => ['P.IVA IT55679497721'], + 'counter_samples' => ['IT00000000001'], + ], + ], + + 'es' => [ + 'es_dni' => [ + 'pattern' => '/\b(?:\d{8}|[XYZ]\d{7})[A-Z]\b/i', + 'validator' => 'dni', + 'entity' => 'national_id', + 'confidence' => 0.8, + 'min_length' => 9, + 'samples' => ['DNI 68334472T', 'NIE X6732518G'], + 'counter_samples' => ['12345678A'], + ], + 'es_vat' => [ + 'pattern' => '/\bES ?[A-Z0-9]\d{7}[A-Z0-9]\b/', + 'validator' => 'vat', + 'entity' => 'vat_number', + 'confidence' => 0.6, + 'min_length' => 11, + 'samples' => ['ESB12345674'], + 'counter_samples' => ['ES12345'], + ], + ], + + 'be' => [ + 'be_national_number' => [ + 'pattern' => '/\b\d{2}\.?\d{2}\.?\d{2}[-.]?\d{3}\.?\d{2}\b/', + 'validator' => 'belgian_national_number', + 'entity' => 'national_id', + 'confidence' => 0.75, + 'min_length' => 11, + 'samples' => ['54.04.11-613.25', 'RRN 85.07.22-005.34'], + 'counter_samples' => ['54.04.11-613.26'], + ], + 'be_vat' => [ + 'pattern' => '/\bBE ?0?\d{9}\b/', + 'validator' => 'vat', + 'entity' => 'vat_number', + 'confidence' => 0.7, + 'min_length' => 11, + 'samples' => ['BTW BE0190125740'], + 'counter_samples' => ['BE0190125741'], + ], + ], + + 'se' => [ + 'se_personnummer' => [ + 'pattern' => '/\b(?:\d{2})?\d{6}[-+]?\d{4}\b/', + 'validator' => 'personnummer', + 'entity' => 'national_id', + 'confidence' => 0.7, + 'min_length' => 10, + 'samples' => ['600112-7239', '19550914-4548'], + 'counter_samples' => ['600112-7238', 'started at 1694600000'], + ], + 'se_vat' => [ + 'pattern' => '/\bSE ?\d{10} ?01\b/', + 'validator' => 'vat', + 'entity' => 'vat_number', + 'confidence' => 0.7, + 'min_length' => 14, + 'samples' => ['SE213467230801'], + 'counter_samples' => ['SE213467230901'], + ], + ], + + 'no' => [ + 'no_fodselsnummer' => [ + 'pattern' => '/\b\d{6} ?\d{5}\b/', + 'validator' => 'fodselsnummer', + 'entity' => 'national_id', + 'confidence' => 0.8, + 'min_length' => 11, + 'samples' => ['25054326869', 'fnr 130876 78770'], + 'counter_samples' => ['25054326868'], + ], + ], + + 'ca' => [ + 'ca_sin' => [ + 'pattern' => '/\b\d{3}[ -]?\d{3}[ -]?\d{3}\b/', + 'validator' => 'sin', + 'keywords' => ['sin', 'social insurance'], + 'entity' => 'national_id', + 'confidence' => 0.7, + 'min_length' => 9, + 'samples' => ['SIN 965-232-432'], + 'counter_samples' => ['SIN 123-456-789', 'ref 965-232-432'], + ], + ], + + 'au' => [ + 'au_tfn' => [ + 'pattern' => '/\b\d{3} ?\d{3} ?\d{2,3}\b/', + 'validator' => 'tfn', + 'keywords' => ['tfn', 'tax file'], + 'entity' => 'national_id', + 'confidence' => 0.7, + 'min_length' => 8, + 'samples' => ['TFN 261 158 631'], + 'counter_samples' => ['TFN 123 456 789'], + ], + ], + + // Members without a public checksum, accepted on format alone... + 'eu' => [ + 'eu_vat' => [ + 'pattern' => '/\b(?:AT|BG|CY|CZ|DK|EE|EL|FI|HR|HU|IE|LT|LU|LV|MT|PL|PT|RO|SI|SK)U?[A-Z0-9]{8,12}\b/', + 'validator' => 'vat', + 'entity' => 'vat_number', + 'confidence' => 0.6, + 'min_length' => 10, + 'samples' => ['ATU12345678', 'PL1234567890', 'IE1234567FA'], + 'counter_samples' => ['XX12345678', 'CY12345678'], + ], + ], + ], + + /* + |-------------------------------------------------------------------------- + | Tokenization + |-------------------------------------------------------------------------- + | + | The `tokenize` operator replaces a value with a token the application + | can exchange back - `alice@customer.com` becomes `tok_email_k4m9rp2xzq`, + | and Redactor::detokenize() turns it back. For the boundary in front of a + | language model: the model reasons about tokens, the application resolves + | them before acting. Originals are kept in the cache store below, + | encrypted with APP_KEY, for `ttl` seconds (null keeps them forever). + | Tokens are derived with the pseudonymization key, so they are stable + | and cannot be guessed. + | + */ + + 'tokenization' => [ + 'store' => env('REDACTOR_TOKEN_STORE'), + 'ttl' => env('REDACTOR_TOKEN_TTL', 86_400), + ], + + /* + | Dispatch a RedactionPerformed event whenever something is redacted. It + | carries the profile, the keys, and counts per rule and per entity - + | never a value - so a listener can feed metrics or an audit trail without + | becoming a leak itself. + */ + 'events' => env('REDACTOR_EVENTS', true), + /* |-------------------------------------------------------------------------- | Redaction Profiles @@ -55,33 +678,97 @@ | Strategies are executed in the order listed below. */ 'strategies' => [ - \Kirschbaum\Redactor\Strategies\SafeKeysStrategy::class, - \Kirschbaum\Redactor\Strategies\BlockedKeysStrategy::class, - \Kirschbaum\Redactor\Strategies\LargeObjectStrategy::class, - \Kirschbaum\Redactor\Strategies\LargeStringStrategy::class, - \Kirschbaum\Redactor\Strategies\RegexPatternsStrategy::class, - \Kirschbaum\Redactor\Strategies\ShannonEntropyStrategy::class, + SafeKeysStrategy::class, + BlockedKeysStrategy::class, + LargeObjectStrategy::class, + LargeStringStrategy::class, + KnownSecretsStrategy::class, + RegexPatternsStrategy::class, + ShannonEntropyStrategy::class, + EntityRecognitionStrategy::class, // inert until recognition.enabled + ], + + /* + | Named entity recognition: names, addresses and organisations in + | free text, which no regex can express. Asks a recogniser that + | speaks Presidio's /analyze contract, so any model behind that + | API works. A model call costs milliseconds where the rules cost + | microseconds, so enable this on profiles used from queues, + | exports and scans - not on the request path. + | + | Gated to values that read as prose between min_length and + | max_length, to the labels listed, and to scores at or above the + | threshold. Offsets are verified against the value before a span + | is replaced. A failing recogniser is skipped, and after + | failure_threshold consecutive failures not asked again for + | cooldown seconds; the output is then rules-only. + | + | With batch on, every prose value in a payload is sent in one + | call before the walk, so a record with fifty free-text fields + | costs one round trip rather than fifty. + */ + 'recognition' => [ + 'enabled' => env('REDACTOR_RECOGNITION', false), + 'driver' => 'presidio', + 'url' => env('REDACTOR_RECOGNITION_URL', 'http://127.0.0.1:5002/analyze'), + 'language' => 'en', + 'entities' => ['PERSON', 'LOCATION', 'ORGANIZATION', 'NRP'], + 'entity_map' => [ + 'PERSON' => 'person', + 'LOCATION' => 'location', + 'ORGANIZATION' => 'organization', + 'NRP' => 'nationality', + ], + 'score_threshold' => 0.6, + 'min_length' => 20, + 'max_length' => 5000, + 'min_words' => 3, + 'timeout' => 2.0, + 'batch' => true, + 'failure_threshold' => 3, + 'cooldown' => 60, + ], + + /* + | The application's own credentials, redacted wherever they appear + | verbatim. `values` are literals; `config` names keys whose string + | leaves are registered at build time - a key pointing at an array + | registers everything under it. Values under 8 characters are + | skipped, as are nulls, so an unset secret never fails the profile. + | Add more at runtime with Redactor::registerSecret(). + */ + 'known_secrets' => [ + 'values' => [], + 'config' => [ + 'app.key', + // 'services.stripe.secret', + // 'database.connections.mysql.password', + ], ], + /* + | Keys whose contents are safe by construction: identifiers, + | timestamps and enumerations. Everything under a safe key is + | preserved as-is, nested structures included, so a free-text + | field must never be listed here however harmless its name. + */ 'safe_keys' => [ // Core identifiers (high frequency) 'id', 'uuid', 'user_id', 'order_id', - 'session_id', 'request_id', + 'trace_id', // Timestamps & metadata (high frequency) 'created_at', 'updated_at', 'timestamp', - // Redactor framework keys (highest frequency) + // Log framework keys (highest frequency) 'level', 'event', - 'message', - 'trace_id', 'channel', 'duration_ms', 'memory_mb', @@ -94,20 +781,26 @@ 'breaker_tripped', 'uncaught', - 'title', + // Enumerations and fixed vocabularies 'type', 'method', - 'path', - 'url', - 'ip', - 'user_agent', 'operation', 'action', - 'source', - 'target', 'version', 'platform', 'environment', + + /* + | Deliberately NOT safe, though earlier versions listed them: + | + | message, title free text, the commonest PII carrier + | url, path query strings carry tokens and emails + | ip, user_agent personal data under GDPR + | source, target free-form, frequently addresses or paths + | session_id was simultaneously listed under + | blocked_keys; safe_keys won, so it was + | never redacted + */ ], 'blocked_keys' => [ @@ -137,27 +830,116 @@ 'pin', ], + /* + | The shared credential and identity rules, in that order. See + | the top of this file for why the order matters and what + | `keywords` does. + */ 'patterns' => [ - // Ordered by frequency and performance (most common/fastest first) - 'email' => '/[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+/', - 'phone_simple' => '/\b\d{3}[.-]?\d{3}[.-]?\d{4}\b/', - 'ssn' => '/\b\d{3}-?\d{2}-?\d{4}\b/', - 'credit_card' => '/\b(?:\d[ -]*?){13,16}\b/', - 'url_with_auth' => '/https?:\/\/[^:\/\s]+:[^@\/\s]+@[^\s]+/', + ...$credentialPatterns, + ...$identityPatterns, ], + /* + | Region packs to add to the patterns above, by key from the + | `regions` section at the top of this file: ['gb', 'nl', 'eu']. + */ + 'regions' => [], + + /* + | Rules that name a location outright. + | + | request.headers.authorization exactly there + | user.*.email any single level between + | **.password at any depth + | users[*].token through a list + | + | Checked before anything else and, when one matches, instead of + | everything else - no key guessing, no scanning of the contents, + | no walk below the matched node. A path says where; every other + | rule in this file is inferring it. + | + | The more specific pattern wins, so declaration order never + | matters, and `preserve` carves an exception out of a broader rule + | without disabling it. + */ + 'paths' => [ + // 'request.headers.authorization' => 'redact', + // 'user.email' => 'surrogate', + // 'debug' => 'preserve', + ], + + /* + | What happens to what the detectors find, by entity. + | + | redact replace with the replacement string (default) + | mask same length, all mask characters + | partial keep the last N characters + | remove delete it + | hash stable keyed token: [email:k4m9rp2xzq] + | surrogate stable fake of the same shape + | preserve detect and report, change nothing + | + | Entity beats the rule that found it, so a policy decision about + | data is not overridden by which regex happened to spot it. + */ + 'operators' => [ + 'default' => 'redact', + 'credit_card' => ['partial' => ['keep' => 4]], + ], + + /* + | Detections scoring below this are ignored. Raise it to quieten a + | noisy profile without weakening any pattern. + */ + 'min_confidence' => env('REDACTOR_MIN_CONFIDENCE', 0.0), + 'replacement' => env('REDACTOR_REPLACEMENT', '[REDACTED]'), 'mark_redacted' => env('REDACTOR_MARK_REDACTED', true), 'track_redacted_keys' => env('REDACTOR_TRACK_KEYS', false), 'non_redactable_object_behavior' => env('REDACTOR_OBJECT_BEHAVIOR', 'preserve'), 'max_value_length' => env('REDACTOR_MAX_VALUE_LENGTH', 5000), + + /* + | What happens to a string over max_value_length. + | + | truncate keep the head, scan it, note what was cut (default) + | redact replace the whole value + | + | The values most often over the limit in a Laravel log are stack + | traces and request bodies - the part the reader needed - so the + | default keeps what it can rather than replacing all of it. + */ + 'large_string_behavior' => env('REDACTOR_LARGE_STRING_BEHAVIOR', 'truncate'), 'redact_large_objects' => env('REDACTOR_LARGE_OBJECTS', true), 'max_object_size' => env('REDACTOR_MAX_OBJECT_SIZE', 100), + /* + | How many levels deep the redactor will walk before replacing the + | rest of the subtree. Guards against cyclic and pathologically + | nested payloads. + */ + 'max_depth' => env('REDACTOR_MAX_DEPTH', 32), + 'shannon_entropy' => [ 'enabled' => env('REDACTOR_SHANNON_ENABLED', true), 'threshold' => env('REDACTOR_SHANNON_THRESHOLD', 4.8), 'min_length' => env('REDACTOR_SHANNON_MIN_LENGTH', 25), + + /* + | Per-alphabet thresholds. A hex digest cannot exceed 4.0 bits + | per character because it only has 16 symbols to draw on, so + | judging it against a base64 threshold guarantees a miss; + | judging base64 against a hex threshold guarantees false + | positives. Remove this block to judge every token against + | the single `threshold` above. + */ + 'charset_thresholds' => [ + 'hex' => 3.0, // max possible 4.0 + 'base64' => 4.5, // max possible 6.0 + 'base64url' => 4.5, // max possible 6.0 + ], + 'exclusion_patterns' => [ '/^https?:\/\//', '/^[\/\\\\].+[\/\\\\]/', @@ -186,15 +968,23 @@ 'enabled' => true, 'strategies' => [ - \Kirschbaum\Redactor\Strategies\SafeKeysStrategy::class, - \Kirschbaum\Redactor\Strategies\BlockedKeysStrategy::class, - \Kirschbaum\Redactor\Strategies\LargeObjectStrategy::class, - \Kirschbaum\Redactor\Strategies\LargeStringStrategy::class, - \Kirschbaum\Redactor\Strategies\RegexPatternsStrategy::class, - \Kirschbaum\Redactor\Strategies\ShannonEntropyStrategy::class, + SafeKeysStrategy::class, + BlockedKeysStrategy::class, + LargeObjectStrategy::class, + LargeStringStrategy::class, + KnownSecretsStrategy::class, + RegexPatternsStrategy::class, + ShannonEntropyStrategy::class, ], - // Minimal safe keys for strict environments + 'known_secrets' => [ + 'values' => [], + 'config' => ['app.key'], + ], + + // Minimal safe keys for strict environments. 'message' is + // excluded: it is free text, which is exactly what strict mode + // exists to inspect. 'safe_keys' => [ 'id', 'uuid', @@ -203,7 +993,6 @@ 'timestamp', 'level', 'event', - 'message', ], // Extended blocked keys @@ -242,14 +1031,18 @@ ], 'patterns' => [ - 'email' => '/[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+/', - 'phone' => '/\+?[\d\s\-\(\)]{7,15}/', - 'ssn' => '/\b\d{3}-?\d{2}-?\d{4}\b/', - 'credit_card' => '/\b(?:\d[ -]*?){13,16}\b/', - 'url_with_auth' => '/https?:\/\/[^:\/\s]+:[^@\/\s]+@[^\s]+/', - 'ipv4' => '/\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b/', - 'uuid' => '/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/i', - 'jwt' => '/^[A-Za-z0-9-_]+\.[A-Za-z0-9-_]+\.[A-Za-z0-9-_]*$/', + ...$credentialPatterns, + ...$identityPatterns, + 'ipv4' => [ + 'pattern' => '/\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b/', + 'entity' => 'ip', + 'confidence' => 0.8, + ], + 'uuid' => [ + 'pattern' => '/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/i', + 'entity' => 'uuid', + 'confidence' => 0.8, + ], ], 'replacement' => '[REDACTED]', @@ -259,6 +1052,7 @@ 'max_value_length' => 1000, // More aggressive 'redact_large_objects' => true, 'max_object_size' => 25, // Smaller objects + 'max_depth' => 16, // Shallower walk for stricter environments 'shannon_entropy' => [ 'enabled' => true, @@ -288,29 +1082,62 @@ | Only strategies that work well with plain text content */ 'strategies' => [ - \Kirschbaum\Redactor\Strategies\RegexPatternsStrategy::class, - \Kirschbaum\Redactor\Strategies\ShannonEntropyStrategy::class, + KnownSecretsStrategy::class, + RegexPatternsStrategy::class, + ShannonEntropyStrategy::class, + ], + + 'known_secrets' => [ + 'values' => [], + 'config' => ['app.key'], ], // No key-based strategies for file scanning 'safe_keys' => [], 'blocked_keys' => [], - // Enhanced patterns for file content detection + /* + | Patterns for file content detection. + | + | Rules that need surrounding context to match confidently declare + | a `capture` group, so the label survives and only the secret is + | replaced: "aws_secret_access_key = [REDACTED]", not "[REDACTED]". + */ 'patterns' => [ - 'email' => '/[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+/', - 'phone_simple' => '/\b\d{3}[.-]?\d{3}[.-]?\d{4}\b/', - 'ssn' => '/\b\d{3}-?\d{2}-?\d{4}\b/', - 'credit_card' => '/\b(?:\d[ -]*?){13,16}\b/', - 'url_with_auth' => '/https?:\/\/[^:\/\s]+:[^@\/\s]+@[^\s]+/', - 'api_key_stripe' => '/sk_(?:test_|live_)[a-zA-Z0-9]{24,}/', - 'api_key_generic' => '/(?:api[_-]?key|access[_-]?token|secret[_-]?key)[\s=:]+[a-zA-Z0-9_-]{16,}/', - 'jwt_token' => '/eyJ[a-zA-Z0-9_-]*\.eyJ[a-zA-Z0-9_-]*\.[a-zA-Z0-9_-]+/', - 'base64_key' => '/(?:key|token|secret)[\s=:]+[A-Za-z0-9+\/]{32,}={0,2}/', - 'aws_access_key' => '/AKIA[0-9A-Z]{16}/', - 'aws_secret_key' => '/[0-9a-zA-Z\/+]{40}/', - 'github_token' => '/gh[pousr]_[A-Za-z0-9_]{36}/', - 'password_assignment' => '/password[\s=:]+[^\s\n\r]+/', + ...$credentialPatterns, + ...$identityPatterns, + + 'api_key_generic' => [ + 'pattern' => '/(?:api[_-]?key|access[_-]?token|secret[_-]?key)([\s=:]+["\']?)([a-zA-Z0-9_\/+-]{16,})/i', + 'capture' => 2, + ], + + /* + | Was '/[0-9a-zA-Z\/+]{40}/', which matches any 40-character + | alphanumeric run: every SHA-1 digest, every base64 chunk, + | every minified identifier. AWS secret keys are now only + | reported next to something that names them. + */ + 'aws_secret_key' => [ + 'pattern' => '/(aws[_\-. ]?(?:secret[_\-. ]?)?access[_\-. ]?key[_\-. ]?(?:id)?["\']?[\s=:]+["\']?)([0-9a-zA-Z\/+]{40})/i', + 'capture' => 2, + ], + + 'base64_key' => [ + 'pattern' => '/(?:key|token|secret)([\s=:]+["\']?)([A-Za-z0-9+\/]{32,}={0,2})/i', + 'capture' => 2, + ], + + 'password_assignment' => [ + // Keep the "password=" label so the finding is readable. + 'pattern' => '/(password["\']?[\s=:]+["\']?)([^\s\n\r"\']+)/i', + 'capture' => 2, + ], + ], + + 'operators' => [ + 'default' => 'redact', + 'credit_card' => ['partial' => ['keep' => 4]], ], 'replacement' => '[REDACTED]', @@ -320,12 +1147,18 @@ 'max_value_length' => null, 'redact_large_objects' => false, 'max_object_size' => 100, + 'max_depth' => 32, // Tuned Shannon entropy for file scanning 'shannon_entropy' => [ 'enabled' => true, 'threshold' => 4.8, // Standard threshold 'min_length' => 25, // Standard minimum length + 'charset_thresholds' => [ + 'hex' => 3.0, + 'base64' => 4.5, + 'base64url' => 4.5, + ], 'exclusion_patterns' => [ '/^https?:\/\//', '/^[\/\\\\].+[\/\\\\]/', @@ -345,6 +1178,110 @@ ], ], + /* + |---------------------------------------------------------------------- + | Observability Profile + |---------------------------------------------------------------------- + | + | For logs and traces you still need to be able to reason about. + | + | Replacing every value with "[REDACTED]" collapses distinct values into + | one, which destroys exactly the questions logs exist to answer: how + | many users hit this, is it always the same account, did this session + | span both services. This profile pseudonymises instead - the same + | input always yields the same stand-in - so counts, joins and traces + | survive while the original values do not. + | + | Requires a pseudonymization key (see above). Without one it degrades + | to plain redaction rather than emitting an unkeyed surrogate. + | + */ + 'observability' => [ + 'enabled' => true, + + 'strategies' => [ + SafeKeysStrategy::class, + BlockedKeysStrategy::class, + KnownSecretsStrategy::class, + RegexPatternsStrategy::class, + ShannonEntropyStrategy::class, + ], + + 'known_secrets' => [ + 'values' => [], + 'config' => ['app.key'], + ], + + 'safe_keys' => [ + 'id', 'uuid', 'user_id', 'order_id', 'request_id', 'trace_id', + 'created_at', 'updated_at', 'timestamp', 'level', 'event', + 'channel', 'duration_ms', 'memory_mb', 'status', 'method', + 'type', 'action', 'operation', 'version', 'environment', + ], + + 'blocked_keys' => [ + 'password', '*token*', '*secret*', 'authorization', 'private_key', + 'client_secret', 'cvv', 'pin', + ], + + 'patterns' => [ + ...$credentialPatterns, + ...$identityPatterns, + 'ipv4' => [ + 'pattern' => '/\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b/', + 'entity' => 'ip', + 'confidence' => 0.8, + ], + ], + + /* + | Emails keep their domain, so "how many distinct users at this + | tenant" still answers correctly. Cards keep their BIN and stay + | Luhn-valid. IPs become a different-but-stable address, so rate + | analysis by source survives. + */ + 'paths' => [ + 'request.headers.authorization' => 'redact', + 'request.headers.cookie' => 'redact', + '**.password' => 'redact', + ], + + 'operators' => [ + 'default' => 'redact', + 'email' => ['surrogate' => ['preserve_domain' => true]], + 'phone' => 'surrogate', + 'ip' => 'surrogate', + 'credit_card' => ['surrogate' => ['preserve_bin' => 6]], + ], + + 'min_confidence' => 0.4, + + 'replacement' => '[REDACTED]', + 'mark_redacted' => false, + 'track_redacted_keys' => false, + 'non_redactable_object_behavior' => 'preserve', + 'max_value_length' => 5000, + 'redact_large_objects' => true, + 'max_object_size' => 100, + 'max_depth' => 32, + + 'shannon_entropy' => [ + 'enabled' => true, + 'threshold' => 4.8, + 'min_length' => 25, + 'charset_thresholds' => [ + 'hex' => 3.0, + 'base64' => 4.5, + 'base64url' => 4.5, + ], + 'exclusion_patterns' => [ + '/^https?:\\/\\//', + '/^\\d{4}-\\d{2}-\\d{2}/', + '/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i', + ], + ], + ], + /* |---------------------------------------------------------------------- | Performance Profile @@ -358,27 +1295,33 @@ 'enabled' => true, 'strategies' => [ - \Kirschbaum\Redactor\Strategies\SafeKeysStrategy::class, - \Kirschbaum\Redactor\Strategies\BlockedKeysStrategy::class, + SafeKeysStrategy::class, + BlockedKeysStrategy::class, // Skip large object/string checks for performance - \Kirschbaum\Redactor\Strategies\RegexPatternsStrategy::class, + KnownSecretsStrategy::class, + RegexPatternsStrategy::class, // Disable shannon entropy for performance ], + 'known_secrets' => [ + 'values' => [], + 'config' => ['app.key'], + ], + + // Same rule as the default profile: identifiers and enumerations + // only, never free text. 'safe_keys' => [ 'id', 'uuid', 'user_id', 'order_id', - 'session_id', 'request_id', + 'trace_id', 'created_at', 'updated_at', 'timestamp', 'level', 'event', - 'message', - 'trace_id', 'channel', 'duration_ms', 'memory_mb', @@ -388,17 +1331,10 @@ 'status', 'breaker_tripped', 'uncaught', - 'title', 'type', 'method', - 'path', - 'url', - 'ip', - 'user_agent', 'operation', 'action', - 'source', - 'target', 'version', 'platform', 'environment', @@ -414,9 +1350,11 @@ 'client_secret', ], - // Minimal, fast patterns only + // Minimal, fast patterns only. Every rule here is gated on a + // literal, so a value without one costs a str_contains() and + // nothing more. 'patterns' => [ - 'email' => '/[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+/', + 'email' => $identityPatterns['email'], 'simple_token' => '/^[A-Za-z0-9]{32,}$/', ], @@ -427,6 +1365,7 @@ 'max_value_length' => null, // Disable 'redact_large_objects' => false, // Disable 'max_object_size' => null, + 'max_depth' => 16, 'shannon_entropy' => [ 'enabled' => false, // Disabled for performance @@ -440,7 +1379,7 @@ |-------------------------------------------------------------------------- | | Register custom strategy classes that can be used in profiles. - | These should implement RedactionStrategyInterface. + | These should implement Strategies\Contracts\Strategy. | */ diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000..6802552 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,38 @@ +# Redactor Documentation + +Redactor removes sensitive data from anything a Laravel application emits: log records, HTTP responses, streams, MCP tool results, prompts sent to a language model, exports and jobs. Detection is a set of rules you configure per profile; what happens to a detected value is a separate, per-entity decision called an operator. + +## Start Here + +If you are new to the package, read the pages in this order: + +1. [Getting Started](getting-started.md) installs the package, adds the log tap and runs your first redaction. +2. [Boundaries](boundaries.md) shows where data leaves an application and which adapter covers each exit. +3. [Configuration](configuration.md) is the reference for every key in `config/redactor.php`. + +Everything else can be read as you need it. + +## Pages + +| Page | What it covers | +| --- | --- | +| [Getting Started](getting-started.md) | Installation, the Monolog tap, `redact()`, `inspect()`, the fluent builder, profiles and the five that ship. | +| [Configuration](configuration.md) | Every top-level and per-profile key with its type, default and environment variable; the shipped profiles compared. | +| [Rules](rules.md) | Pattern rules, validators, keywords, samples, dictionary rules, allow-lists, path rules, safe and blocked keys, known secrets, confidence and how detections are resolved. | +| [Operators and Pseudonymisation](operators-and-pseudonymisation.md) | Every operator with example output, precedence, surrogates, the key and salt, reversible tokens and `detokenize()`. | +| [Boundaries](boundaries.md) | Log channels, HTTP responses, streams, MCP servers, AI agents, exports and jobs, and the `RedactionPerformed` event. | +| [Scanning](scanning.md) | `redactor:scan` in full: paths, output formats, git modes, decoding, baselines, inline suppression, verification, the pre-commit hook and exit codes. | +| [Entity Recognition](entity-recognition.md) | Finding names, places and organisations in prose with a Presidio-compatible recogniser or the in-process `redactor-onnx` package, and when not to. | +| [Testing](testing.md) | `Redactor::fake()` and its assertions, `redactor:validate`, rule samples, and the package's own test conventions. | +| [Extending](extending.md) | Every contract, how to register each, a worked custom strategy and operator, and macros. | +| [Upgrading](upgrading.md) | Every renamed class and method from 0.1.0, every behaviour change, and what to do about each. | + +## Conventions + +Code samples assume the facade is imported: + +```php +use Kirschbaum\Redactor\Facades\Redactor; +``` + +Configuration paths are written relative to the file, so `profiles.default.patterns` means `config('redactor.profiles.default.patterns')`. diff --git a/docs/boundaries.md b/docs/boundaries.md new file mode 100644 index 0000000..5315b39 --- /dev/null +++ b/docs/boundaries.md @@ -0,0 +1,275 @@ +# Boundaries + +- [Introduction](#introduction) +- [Log Channels](#log-channels) + - [The Tap](#the-tap) + - [The Formatter](#the-formatter) + - [Opaque Objects](#opaque-objects) + - [Long Strings and Large Objects](#long-strings-and-large-objects) +- [HTTP Responses](#http-responses) +- [Streams](#streams) +- [MCP Servers](#mcp-servers) +- [AI Agents](#ai-agents) +- [Exports, Jobs and Everything Else](#exports-jobs-and-everything-else) +- [Events](#events) + +## Introduction + +Data leaves an application through more doors than the log. Each one below has an adapter that applies a profile at the boundary, so the same rules hold wherever a value goes and the profile name is the only thing that changes. + +| Boundary | Adapter | Profile chosen by | +| --- | --- | --- | +| Log channel | `Logging\RedactorTap` | the tap's argument | +| HTTP response | `redact` middleware | the middleware's argument | +| Streamed output | `Streaming\StreamRedactor` | the constructor | +| MCP server | `Mcp\RedactsResponses` | `redactionProfile()` | +| AI agent prompt | `Ai\RedactPrompt` | `RedactPrompt::using()` | +| Anything else | `Redactor::redact()` | the second argument | + +Every adapter uses `redactSafely()`, so a failure replaces the content rather than letting it through or throwing. + +## Log Channels + +### The Tap + +Add `RedactorTap` to any channel in `config/logging.php`: + +```php +'single' => [ + 'driver' => 'single', + 'path' => storage_path('logs/laravel.log'), + 'level' => env('LOG_LEVEL', 'debug'), + 'tap' => [Kirschbaum\Redactor\Logging\RedactorTap::class], +], + +'audit' => [ + 'driver' => 'daily', + 'path' => storage_path('logs/audit.log'), + 'tap' => [Kirschbaum\Redactor\Logging\RedactorTap::class.':strict'], +], +``` + +The tap pushes a `RedactorProcessor` onto the channel's Monolog logger. The processor redacts the record's message, context and extra, and then the channel's own formatter renders the result, so a channel writing JSON keeps writing JSON. Put the tap on a `stack` channel and every channel in the stack is covered. + +Redaction inside the processor never throws. A profile name typo, an unreadable config value or a strategy that fails on unexpected input all fail closed: the message becomes `[REDACTED] (redaction failed)`, the context and extra become `['redaction' => '...']`, and the record is still written. The package's own diagnostics go through a re-entrancy guard, so a warning raised inside the log pipeline cannot re-enter the handler that triggered it. + +Run `php artisan redactor:validate` at deploy time to find a broken profile before the first log line does. + +### The Formatter + +`RedactorFormatter` is a Monolog formatter that redacts and renders. It owns the output format, so prefer the tap unless a channel specifically wants a self-contained drop-in. It can wrap an inner formatter rather than replace it: + +```php +use Kirschbaum\Redactor\Logging\RedactorFormatter; +use Monolog\Formatter\JsonFormatter; + +$handler->setFormatter(new RedactorFormatter(new JsonFormatter)); +``` + +Without an inner formatter it writes `[datetime] channel.LEVEL: message {context} {extra}`. `formatBatch()` renders every record, so batching handlers lose nothing. + +`RedactorFormatterTap` installs `RedactorFormatter` on every formattable handler of a channel. It discards whatever formatter the handler had, so a channel writing JSON stops writing JSON the moment it is enabled. It is the successor of the 0.1.0 `CustomLogTap` and is kept for channels that relied on it; new channels should use `RedactorTap`. + +### Opaque Objects + +Throwables, `DateTimeInterface`, `DateTimeZone`, enums and closures pass through the walk untouched. A Throwable has no public properties and would encode to `{}`, so `['exception' => $e]` reaching the formatter as `[]` would lose the stack trace; a Carbon instance would be exploded into its `toArray()` components. Every logging formatter already knows how to render them. + +Key rules still apply, since the strategy chain runs before the opacity check: + +```php +Log::error('failed', ['exception' => $e, 'password' => 'x']); +// ['exception' => $e, 'password' => '[REDACTED]'] + +Log::info('state', ['secret' => SomeEnum::Value]); +// ['secret' => '[REDACTED]'] +``` + +Other objects are walked. Anything with a `toArray()` method, a model or a collection, is walked through that; anything else is walked through its JSON encoding. An object that can be neither converted is handled according to `non_redactable_object_behavior`: `preserve` (default), `remove`, `redact` or `empty_array`. + +### Long Strings and Large Objects + +The values most often over `max_value_length` in a Laravel log are stack traces and request bodies, which is exactly what the reader needed, so the default keeps the head, scans it, and notes what was cut: + +``` +Stack trace: #0 /app/Http/... [REDACTED] (String truncated: 65536 characters, 5000 kept) +``` + +Set `large_string_behavior` to `redact` to replace the whole value instead. An array or object with more items than `max_object_size` becomes `['_large_object_redacted' => '[REDACTED] (Array with 250 items)']`. A subtree deeper than `max_depth` becomes `[REDACTED] (Max depth of 32 exceeded)`, and an object already on the recursion stack becomes `[REDACTED] (Circular reference to App\Models\User)`. + +## HTTP Responses + +Data that leaves through an API needs the same boundary as data that leaves through a log. The package registers a `redact` middleware alias that redacts a response before it is sent, with a profile per route: + +```php +Route::get('/export', ExportController::class)->middleware('redact:observability'); +Route::get('/me', MeController::class)->middleware('redact'); +``` + +What happens depends on the response: + +| Response | Treatment | +| --- | --- | +| `JsonResponse` | Redacted as data, so structure and types survive. | +| Any response whose `Content-Type` contains `json` | Decoded, redacted as data, re-encoded. Falls back to text if the body is not valid JSON. | +| `text/*`, `xml`, `javascript`, `x-www-form-urlencoded`, or no content type | Redacted as text. | +| `StreamedResponse` | The callback is wrapped in a `StreamRedactor`. See [Streams](#streams). | +| `BinaryFileResponse`, any other content type, empty body | Passed through. | + +The profile's `_redacted` markers are never written into a response. Where a consumer has typed the field, use `nullify` so the field stays a field and keeps its type: + +```php +'operators' => ['ssn' => 'nullify', 'age' => 'nullify'], +``` + +The middleware fails closed. A response that cannot be redacted becomes a JSON `500` with the body `{"message": "The response could not be redacted."}` and none of the original content. Run `redactor:validate` at deploy time so that never happens in production. + +## Streams + +A streamed response, server-sent events, a model's tokens, a file piped through, is redacted as it streams. The `redact` middleware wraps a `StreamedResponse` callback for you; for anything else, use `StreamRedactor` directly: + +```php +use Kirschbaum\Redactor\Redactor; +use Kirschbaum\Redactor\Streaming\StreamRedactor; + +$stream = new StreamRedactor(app(Redactor::class), 'observability'); + +// An iterable of chunks: a generator, an array, a model's token stream +foreach ($stream->through($llm->tokens()) as $safe) { + echo $safe; +} + +// A callback that echoes, wrapped so what it echoes is redacted +$callback = $stream->wrap(fn () => $this->export($rows)); + +// The same, as a StreamedResponse +return $stream->response(fn () => $this->export($rows), 200, ['Content-Type' => 'text/csv']); +``` + +For full control, feed chunks yourself: + +```php +echo $stream->push($chunk); // returns whatever is now safe to emit, possibly '' +echo $stream->flush(); // once the stream has ended +``` + +Redacting each chunk on its own would miss every secret that straddles a boundary, and buffering the whole stream would defeat the point of streaming. So `StreamRedactor` holds back the most recent bytes until more arrive: + +- The last `holdback` bytes (the third constructor argument, default 1024) are never emitted until more input pushes them out. +- The cut always falls on a line end. No shipped rule except the PEM block matches across a newline, so a line end is always safe. +- When a stream goes a whole hold-back window without a line end, the cut falls on a word boundary instead. With no boundary at all, an unbroken run is emitted once it has outlived a window. +- A PEM block that has opened but not closed is held whole, however many lines it spans. + +The cost is latency in bytes, not time. Raise the hold-back for content whose secrets are longer than a screen line. `wrap()` and `response()` take a `$chunkSize` (default 4096) for the output buffer that feeds the redactor. + +## MCP Servers + +An MCP server hands data straight to a model. With Laravel's MCP package, one trait redacts everything the server returns: + +```php +use Kirschbaum\Redactor\Mcp\RedactsResponses; +use Laravel\Mcp\Server; + +class SupportServer extends Server +{ + use RedactsResponses; + + protected function redactionProfile(): ?string + { + return 'observability'; // null for the default profile + } +} +``` + +The trait sits where the server builds its JSON-RPC responses, so it covers HTTP and stdio alike, and the server's own test helpers exercise it: `SupportServer::tool(LookupCustomer::class)->assertDontSee($email)` tests the redaction along with the tool. + +What is redacted: + +| Part of the response | Treatment | +| --- | --- | +| `result.content[].text` (tool results) | As text. | +| `result.content[].resource.text` (embedded resources) | As text. | +| `result.structuredContent` | As data, without markers. If it cannot be redacted it becomes `['redaction' => 'failed']`. | +| `result.contents[]` (resource reads) | As text. | +| `result.messages[].content` (prompt messages) | As text, whether a string or a content block. | +| `error.message` | As text. | +| `params.content` of a streamed notification | As tool output. | + +What is not: the JSON-RPC envelope, ids, protocol fields, and any binary content such as an image's `data` or a `blob`, so the protocol stays valid and an image is not mistaken for a high-entropy secret. + +Use a profile whose operators tokenise when the model needs to refer to a value the application will act on. See [Reversible Tokens](operators-and-pseudonymisation.md#reversible-tokens). `laravel/mcp` is a suggested dependency, not a required one. + +## AI Agents + +With Laravel's AI package, the `RedactPrompt` middleware redacts a prompt before the provider sees it and resolves tokens in the answer on the way back: + +```php +use Kirschbaum\Redactor\Ai\RedactPrompt; +use Laravel\Ai\Contracts\Agent; +use Laravel\Ai\Contracts\HasMiddleware; +use Laravel\Ai\Promptable; + +class SupportAgent implements Agent, HasMiddleware +{ + use Promptable; + + public function instructions(): string + { + return 'You answer support tickets.'; + } + + public function middleware(): array + { + return [RedactPrompt::using('ai')]; + } +} +``` + +`RedactPrompt::using(?string $profile, bool $detokenizeResponse = true)` takes the profile and whether to resolve tokens in the response text. With a profile whose operators tokenise, the model reasons about `tok_email_k4m9rp2xzq` and the application receives the real address back in the response. With a profile that redacts outright, the model never sees the value. Pass `detokenizeResponse: false` to keep tokens in the answer, for instance when the answer is going to be logged or shown rather than acted on. + +The round trip is: + +1. The prompt text is passed through `redactSafely()` with the profile. +2. The revised prompt goes to the provider. +3. When the response arrives, `Redactor::detokenize()` replaces every known token in its text. + +`laravel/ai` is a suggested dependency, not a required one. + +## Exports, Jobs and Everything Else + +The same call covers everything else an application emits. Every path accepts a profile name, so the same value can be pseudonymised on one channel and removed on another: + +```php +// A queued export, on a profile that pseudonymises +ExportRow::create(Redactor::redact($user->toArray(), 'observability')); + +// A support transcript before it reaches a third party +$client->createTicket(Redactor::redact($conversation, 'strict')); + +// An error reporter's outgoing payload, in whichever hook it offers +$reporter->beforeSend(fn (array $event) => Redactor::redactSafely($event, 'strict')); + +// A debug endpoint +return response()->json(Redactor::redact($state, 'performance')); +``` + +Use `redactSafely()` inside someone else's error path. It never throws and fails closed, which is what a hook that runs while an error is already being reported needs. + +## Events + +Every redaction that changed something dispatches `Kirschbaum\Redactor\Events\RedactionPerformed`: + +```php +use Kirschbaum\Redactor\Events\RedactionPerformed; + +Event::listen(RedactionPerformed::class, function (RedactionPerformed $event) { + $event->profile; // 'default' + $event->redactedKeys; // ['password', 'email'], deduplicated + $event->rules; // ['blocked_key' => 2, 'email' => 1] + $event->entities; // ['password' => 1, 'email' => 2] + $event->findings; // 3 +}); +``` + +The event carries names and counts only, never a value, so a listener that writes to metrics or an audit trail cannot itself become the leak. A listener that throws never breaks the redaction; the failure is logged and the redacted value is returned as normal. + +Set `REDACTOR_EVENTS=false`, or `redactor.events` to false, to switch dispatching off. diff --git a/docs/configuration.md b/docs/configuration.md new file mode 100644 index 0000000..5e68ec4 --- /dev/null +++ b/docs/configuration.md @@ -0,0 +1,347 @@ +# Configuration + +- [Introduction](#introduction) +- [Top-Level Keys](#top-level-keys) + - [default_profile](#default_profile) + - [scan](#scan) + - [pseudonymization](#pseudonymization) + - [tokenization](#tokenization) + - [events](#events) + - [profiles](#profiles) + - [custom_strategies](#custom_strategies) +- [Profile Keys](#profile-keys) + - [Strategies and Keys](#strategies-and-keys) + - [Rules](#rules) + - [Operators and Confidence](#operators-and-confidence) + - [Output and Limits](#output-and-limits) + - [shannon_entropy](#shannon_entropy) + - [recognition](#recognition) + - [known_secrets](#known_secrets) + - [pseudonymization (per profile)](#pseudonymization-per-profile) +- [The Shared Pattern Lists](#the-shared-pattern-lists) +- [The Shipped Profiles Compared](#the-shipped-profiles-compared) +- [Environment Variables](#environment-variables) +- [How Values Are Validated](#how-values-are-validated) + +## Introduction + +All of the package's configuration lives in `config/redactor.php`. Publish it with `php artisan vendor:publish --tag=redactor-config`. + +The file has two parts. The top of the file defines two pattern lists, `$credentialPatterns` and `$identityPatterns`, and the returned array spreads them into each profile. The returned array holds the global settings and the profiles. + +Resolved profiles are cached and rebuilt only when the raw configuration behind them changes, so reading a profile on every redaction costs nothing measurable. Invalid values throw a `ConfigurationException` naming the offending path rather than falling back to a default silently. + +## Top-Level Keys + +### default_profile + +| Type | Default | Environment variable | +| --- | --- | --- | +| `string` | `'default'` | `REDACTOR_DEFAULT_PROFILE` | + +The profile used when none is named. It must be a key under `profiles`. + +### scan + +Settings for `redactor:scan`. None of them affect redaction of live payloads. + +| Key | Type | Default | Env | Meaning | +| --- | --- | --- | --- | --- | +| `profile` | `string` | `'file_scan'` | `REDACTOR_SCAN_PROFILE` | The profile the scanner uses unless `--profile` is passed. | +| `exclude_patterns` | `string[]` | `*.lock`, `*.min.js`, `*.map`, `vendor/*`, `node_modules/*`, `storage/framework/*`, `public/build/*` | | Globs matched against each file's basename and its path relative to the scanned directory. A pattern ending in `/*` prunes that directory during the walk. | +| `max_file_size` | `int` | `10485760` | `REDACTOR_SCAN_MAX_FILE_SIZE` | Files larger than this many bytes are skipped. | +| `skip_binary` | `bool` | `true` | `REDACTOR_SCAN_SKIP_BINARY` | Skip files that contain a NUL byte or are mostly non-printable in their first 8 KB. | +| `respect_gitignore` | `bool` | `true` | `REDACTOR_SCAN_RESPECT_GITIGNORE` | Skip files git already ignores. | +| `window_lines` | `int` | `512` | `REDACTOR_SCAN_WINDOW_LINES` | How many lines are scanned at once. | +| `overlap_lines` | `int` | `4` | `REDACTOR_SCAN_OVERLAP_LINES` | How many lines each window shares with the previous one, so a secret spanning a boundary is still found. | +| `decode` | `bool` | `true` | `REDACTOR_SCAN_DECODE` | Look one layer deep inside base64, percent-encoded and JSON-escaped spans. | +| `verification.enabled` | `bool` | `false` | `REDACTOR_SCAN_VERIFY` | Allow `--verify` to contact providers. | +| `verification.verifiers` | `string[]` | `[]` | | The verifiers permitted to run: `github_token`, `stripe_key`, `slack_token`. An empty list means none. | +| `baseline` | `string\|null` | `base_path('.redactor-baseline.json')` | `REDACTOR_SCAN_BASELINE` | The file of accepted findings. | + +See [Scanning](scanning.md) for what each of these does in practice. + +### pseudonymization + +Settings for the `hash`, `surrogate` and `tokenize` operators. + +| Key | Type | Default | Env | Meaning | +| --- | --- | --- | --- | --- | +| `enabled` | `bool` | `true` | `REDACTOR_PSEUDONYMIZATION` | When false the pseudonymising operators fall back to plain redaction. | +| `key` | `string\|null` | `null` | `REDACTOR_PSEUDONYMIZATION_KEY` | The HMAC key. At least 16 bytes. Leave null to derive one from `APP_KEY`. | +| `salt` | `string\|null` | `null` | `REDACTOR_PSEUDONYMIZATION_SALT` | Mixed into every surrogate. Shared by every profile unless a profile sets its own. | + +The mapping is one-way. Anyone holding the key can confirm a guess, so the key must not travel with the logs. Rotating it changes every surrogate. See [Operators and Pseudonymisation](operators-and-pseudonymisation.md#the-key-and-the-salt). + +### tokenization + +Settings for the `tokenize` operator and `Redactor::detokenize()`. + +| Key | Type | Default | Env | Meaning | +| --- | --- | --- | --- | --- | +| `store` | `string\|null` | `null` | `REDACTOR_TOKEN_STORE` | The cache store that holds originals. Null means the default cache store. | +| `ttl` | `int\|null` | `86400` | `REDACTOR_TOKEN_TTL` | How many seconds a token can be exchanged back. Null keeps originals forever. | + +Originals are encrypted with the application key before they reach the cache. See [Reversible Tokens](operators-and-pseudonymisation.md#reversible-tokens). + +### events + +| Type | Default | Environment variable | +| --- | --- | --- | +| `bool` | `true` | `REDACTOR_EVENTS` | + +Whether to dispatch `RedactionPerformed` when a redaction changes something. See [Events](boundaries.md#events). + +### profiles + +An array of named profiles. Every key a profile accepts is described under [Profile Keys](#profile-keys). + +### custom_strategies + +| Type | Default | +| --- | --- | +| `array` | `[]` | + +Strategy classes registered under a short name, so a profile's `strategies` list can name them: + +```php +'custom_strategies' => [ + 'internal_data' => \App\Redaction\InternalDataStrategy::class, +], +``` + +Each class must implement `Kirschbaum\Redactor\Strategies\Contracts\Strategy`. See [Extending](extending.md#strategies). + +## Profile Keys + +Every profile accepts the keys below. Where the shipped `default` profile reads an environment variable, it is listed; the other profiles set literal values. The "Default" column is what applies when the key is absent from a profile, which is not always what the shipped profiles set. + +### Strategies and Keys + +| Key | Type | Default | Env | Meaning | +| --- | --- | --- | --- | --- | +| `enabled` | `bool` | `true` | `REDACTOR_ENABLED` | When false, `redact()` returns the content unchanged and `inspect()` reports nothing. | +| `strategies` | `string[]` | `[]` | | Strategy class names or `custom_strategies` names, in the order they run. | +| `safe_keys` | `string[]` | `[]` | | Keys whose values, subtrees included, are emitted untouched. Supports `*` wildcards, compared case-insensitively. | +| `blocked_keys` | `string[]` | `[]` | | Keys whose values are always redacted. Same wildcard syntax. | + +### Rules + +| Key | Type | Default | Meaning | +| --- | --- | --- | --- | +| `patterns` | `array` | `[]` | Named pattern rules, shorthand regex or full form. See [Rules](rules.md#pattern-rules). | +| `paths` | `array` | `[]` | Dotted path patterns mapped to an operator. See [Path Rules](rules.md#path-rules). | +| `allowlist` | `string[]` | `[]` | Literals and regexes that are never findings whichever detector reports them. See [Allow-Lists](rules.md#allow-lists). | +| `known_secrets` | `array` | `[]` | The application's own credentials. See [known_secrets](#known_secrets). | + +### Operators and Confidence + +| Key | Type | Default | Env | Meaning | +| --- | --- | --- | --- | --- | +| `operators` | `array` | `[]` | | What to do with each entity, plus a `default`. When absent the default operator is `redact`. See [Operators](operators-and-pseudonymisation.md). | +| `min_confidence` | `float` | `0.0` | `REDACTOR_MIN_CONFIDENCE` | Detections scoring below this are ignored. Must be between 0 and 1. See [Confidence](rules.md#confidence). | + +### Output and Limits + +| Key | Type | Default | Env | Meaning | +| --- | --- | --- | --- | --- | +| `replacement` | `string` | `'[REDACTED]'` | `REDACTOR_REPLACEMENT` | The text the `redact` operator writes. | +| `mark_redacted` | `bool` | `true` | `REDACTOR_MARK_REDACTED` | Add `'_redacted' => true` to an associative array that was changed. Never added to a list, never overwrites an existing `_redacted` key, never written into an HTTP response or MCP structured content. | +| `track_redacted_keys` | `bool` | `false` | `REDACTOR_TRACK_KEYS` | With `mark_redacted`, also add `_redacted_keys`. | +| `non_redactable_object_behavior` | `string` | `'preserve'` | `REDACTOR_OBJECT_BEHAVIOR` | What to do with an object that cannot be walked: `preserve`, `remove`, `redact` or `empty_array`. | +| `max_value_length` | `int\|null` | `null` | `REDACTOR_MAX_VALUE_LENGTH` | Strings longer than this many bytes are truncated or redacted. Null disables the check. | +| `large_string_behavior` | `string` | `'truncate'` | `REDACTOR_LARGE_STRING_BEHAVIOR` | `truncate` keeps the head, scans it and appends a note; `redact` replaces the whole value. | +| `redact_large_objects` | `bool` | `true` | `REDACTOR_LARGE_OBJECTS` | Whether `LargeObjectStrategy` does anything. | +| `max_object_size` | `int\|null` | `100` | `REDACTOR_MAX_OBJECT_SIZE` | Arrays and objects with more items than this are replaced with a summary. Null disables the check. | +| `max_depth` | `int` | `32` | `REDACTOR_MAX_DEPTH` | How many levels the walk descends before replacing the rest of the subtree. Guards cyclic and pathologically nested payloads. | + +A truncated string looks like this: + +``` + [REDACTED] (String truncated: 65536 characters, 5000 kept) +``` + +The head is cut with `mb_strcut()` so it stays valid UTF-8, and the strategies after `LargeStringStrategy` still scan it. + +### shannon_entropy + +| Key | Type | Default | Env | Meaning | +| --- | --- | --- | --- | --- | +| `enabled` | `bool` | off when absent | `REDACTOR_SHANNON_ENABLED` | Whether the entropy detector runs. The strategy treats a missing key as disabled. | +| `threshold` | `float` | `4.8` | `REDACTOR_SHANNON_THRESHOLD` | Bits per character a token must reach, unless a charset threshold applies. | +| `min_length` | `int` | `25` | `REDACTOR_SHANNON_MIN_LENGTH` | Tokens shorter than this, in characters, are never measured. | +| `charset_thresholds` | `array` | none | | Per-alphabet thresholds for `hex`, `base64` and `base64url`. When the token's alphabet has one, it wins over `threshold`. | +| `exclusion_patterns` | `string[]` | `[]` | | Regexes for tokens that score high without being sensitive: URLs, dates, UUIDs, IPs, MAC addresses, SQL. A pattern that cannot be evaluated excuses nothing. | + +A hex digest cannot exceed 4.0 bits per character because it has 16 symbols, so judging it against 4.8 guarantees a miss. The shipped profiles set `hex` to 3.0 and both base64 alphabets to 4.5. The hex exclusion `/^[0-9a-f]+$/i` deliberately does not excuse strings of 32 characters or more, since those may be digests. + +Tokens are whitespace-delimited. A value with no internal whitespace is one token, so a bare API key is reported whole; a sentence with a key in it reports only the key. + +### recognition + +Named entity recognition. Present in the shipped `default` profile and inert until enabled. + +| Key | Type | Default | Env | Meaning | +| --- | --- | --- | --- | --- | +| `enabled` | `bool` | `false` | `REDACTOR_RECOGNITION` | Whether `EntityRecognitionStrategy` does anything. When false the strategy is left out of the chain entirely. | +| `driver` | `string` | `'presidio'` | | The registered recogniser to use. | +| `url` | `string` | `'http://127.0.0.1:5002/analyze'` | `REDACTOR_RECOGNITION_URL` | The Presidio `/analyze` endpoint. Only read by the `presidio` driver. | +| `language` | `string` | `'en'` | | Passed to the recogniser. | +| `entities` | `string[]` | `[]` (all) | | The recogniser's own labels to ask for. The shipped profile lists `PERSON`, `LOCATION`, `ORGANIZATION`, `NRP`. | +| `entity_map` | `array` | `[]` | | Recogniser label to package entity, for `operators`. An unmapped label is lowercased. | +| `score_threshold` | `float` | `0.6` | | Spans scoring below this are dropped. | +| `min_length` | `int` | `20` | | Values shorter than this many bytes are not sent. | +| `max_length` | `int` | `5000` | | Values longer than this are not sent. | +| `min_words` | `int` | `3` | | Values with fewer whitespace-separated words are not sent. | +| `timeout` | `float` | `2.0` | | Seconds to wait for the recogniser. | +| `batch` | `bool` | `true` | | Send every prose value in a payload in one call before the walk, instead of one call per value. | +| `failure_threshold` | `int` | `3` | | Consecutive failures before the circuit breaker opens. | +| `cooldown` | `int` | `60` | | Seconds the breaker stays open. | + +See [Entity Recognition](entity-recognition.md). + +### known_secrets + +| Key | Type | Default | Meaning | +| --- | --- | --- | --- | +| `values` | `string[]` | `[]` | Literal secrets. | +| `config` | `string[]` | `[]` | Config keys whose string values are secrets. A key that points at an array registers every string under it. | + +The shipped profiles list `app.key`. Values shorter than 8 characters and nulls are skipped, so an unset secret in a local environment never fails the profile. See [Known Secrets](rules.md#known-secrets). + +### pseudonymization (per profile) + +A profile may carry its own `pseudonymization` block. Its non-null keys are merged over the global block, so a profile can set a `salt` of its own to stop its surrogates correlating with other profiles, or set `enabled` to false: + +```php +'export' => [ + 'pseudonymization' => ['salt' => 'export-2026'], + // ... +], +``` + +## The Shared Pattern Lists + +The two lists at the top of the config file are spread into the `default`, `strict`, `observability` and `file_scan` profiles. Order matters: on an equal confidence score the rule listed first wins an overlap, which is why `url_with_auth` sits ahead of `email` and `anthropic_key` ahead of `openai_key`. + +`$credentialPatterns`, in order: + +| Rule | Entity | Confidence | Notes | +| --- | --- | --- | --- | +| `url_with_auth` | `url_credentials` | 0.9 | Any scheme. Only the password is replaced (`capture` 2). | +| `private_key_block` | `private_key` | 1.0 | PEM `BEGIN ... PRIVATE KEY` to `END`, across lines. | +| `jwt` | `jwt` | 0.9 | Three base64url segments, the first two starting `eyJ`. | +| `bearer_token` | `bearer_token` | 0.85 | `Bearer `; only the token is replaced. | +| `aws_access_key` | `aws_access_key` | 0.9 | `AKIA` or `ASIA` plus 16 characters. | +| `github_token` | `github_token` | 0.95 | `ghp_`, `gho_`, `ghu_`, `ghs_`, `ghr_` and `github_pat_` tokens. | +| `stripe_key` | `stripe_key` | 0.95 | `sk_` and `rk_` keys only; publishable keys are meant to be seen. | +| `slack_token` | `slack_token` | 0.9 | `xox[abpors]-` tokens. | +| `anthropic_key` | `anthropic_key` | 0.95 | `sk-ant-` keys. | +| `openai_key` | `openai_key` | 0.9 | `sk-` and `sk-proj-` keys. | +| `google_api_key` | `google_api_key` | 0.9 | `AIza` plus 35 characters. | +| `sendgrid_key` | `sendgrid_key` | 0.95 | `SG.` keys. | + +`$identityPatterns`, in order: + +| Rule | Entity | Confidence | Notes | +| --- | --- | --- | --- | +| `email` | `email` | 0.8 | Byte-level, so non-ASCII local parts and domains match. Keyword `@`. | +| `phone_formatted` | `phone` | 0.6 | Needs separators or parentheses, so dates, versions and cards are not mistaken. | +| `phone_e164` | `phone` | 0.7 | `+` and 9 to 15 digits. | +| `phone_bare` | `phone` | 0.5 | Ten bare digits, only when the value contains `phone`, `tel`, `mobile`, `cell` or `fax`. | +| `ssn` | `ssn` | 0.7 | Hyphenated, with the `ssn` validator. | +| `ssn_bare` | `ssn` | 0.4 | Nine bare digits, only near `ssn`, `social security`, `tax id` or `tin`, with the validator. | +| `credit_card` | `credit_card` | 0.6 (default) | 13 to 16 digits with optional spaces or dashes, with the `luhn` validator. | +| `iban` | `iban` | 0.6 (default) | Compact or spaced, with the `iban` validator. | + +Every rule in both lists declares `samples`, `counter_samples` and `min_length`, and every rule that can carries `keywords`. See [Rules](rules.md). + +## The Shipped Profiles Compared + +| Setting | `default` | `strict` | `observability` | `file_scan` | `performance` | +| --- | --- | --- | --- | --- | --- | +| Strategies | Safe, Blocked, LargeObject, LargeString, KnownSecrets, Regex, Entropy, Recognition | Safe, Blocked, LargeObject, LargeString, KnownSecrets, Regex, Entropy | Safe, Blocked, KnownSecrets, Regex, Entropy | KnownSecrets, Regex, Entropy | Safe, Blocked, KnownSecrets, Regex | +| Safe keys | 27 identifiers, timestamps and enumerations | 7 (`id`, `uuid`, `created_at`, `updated_at`, `timestamp`, `level`, `event`) | 21 | none | same 27 as `default` | +| Blocked keys | 24, including `*token*`, `*key*`, `*secret*`, `email`, names, `ssn`, card fields | `default` plus `secret`, `phone`, `address`, `user_agent`, `ip`, `name`, `username` | 8 (`password`, `*token*`, `*secret*`, `authorization`, `private_key`, `client_secret`, `cvv`, `pin`) | none | 7 (`password`, `secret`, `*token*`, `*key*`, `authorization`, `private_key`, `client_secret`) | +| Patterns | shared lists | shared lists, `ipv4`, `uuid` | shared lists, `ipv4` | shared lists, `api_key_generic`, `aws_secret_key`, `base64_key`, `password_assignment` | `email`, `simple_token` | +| Paths | none | none | `request.headers.authorization`, `request.headers.cookie`, `**.password` all `redact` | none | none | +| Operators | `credit_card` partial keep 4 | none configured (all `redact`) | `email` surrogate keeping domain, `phone` and `ip` surrogate, `credit_card` surrogate keeping 6-digit BIN | `credit_card` partial keep 4 | none configured | +| `min_confidence` | 0.0 | 0.0 | 0.4 | 0.0 | 0.0 | +| `mark_redacted` | true | true | false | true | false | +| `track_redacted_keys` | false | true | false | false | false | +| `non_redactable_object_behavior` | preserve | redact | preserve | preserve | preserve | +| `max_value_length` | 5000 | 1000 | 5000 | null | null | +| `redact_large_objects` | true | true | true | false | false | +| `max_object_size` | 100 | 25 | 100 | 100 | null | +| `max_depth` | 32 | 16 | 32 | 32 | 16 | +| Entropy | on, 4.8 over 25, charset thresholds | on, 4.0 over 15, no charset thresholds | on, 4.8 over 25, charset thresholds | on, 4.8 over 25, charset thresholds, extra word and number exclusions | off | +| Known secrets | `app.key` | `app.key` | `app.key` | `app.key` | `app.key` | +| Recognition | present, disabled | not listed | not listed | not listed | not listed | + +## Environment Variables + +Every variable the shipped configuration reads: + +```env +# Global +REDACTOR_DEFAULT_PROFILE=default +REDACTOR_EVENTS=true + +# Pseudonymisation and tokens +REDACTOR_PSEUDONYMIZATION=true +REDACTOR_PSEUDONYMIZATION_KEY= +REDACTOR_PSEUDONYMIZATION_SALT= +REDACTOR_TOKEN_STORE= +REDACTOR_TOKEN_TTL=86400 + +# The default profile +REDACTOR_ENABLED=true +REDACTOR_REPLACEMENT="[REDACTED]" +REDACTOR_MARK_REDACTED=true +REDACTOR_TRACK_KEYS=false +REDACTOR_OBJECT_BEHAVIOR=preserve +REDACTOR_MAX_VALUE_LENGTH=5000 +REDACTOR_LARGE_STRING_BEHAVIOR=truncate +REDACTOR_LARGE_OBJECTS=true +REDACTOR_MAX_OBJECT_SIZE=100 +REDACTOR_MAX_DEPTH=32 +REDACTOR_MIN_CONFIDENCE=0.0 +REDACTOR_SHANNON_ENABLED=true +REDACTOR_SHANNON_THRESHOLD=4.8 +REDACTOR_SHANNON_MIN_LENGTH=25 +REDACTOR_RECOGNITION=false +REDACTOR_RECOGNITION_URL=http://127.0.0.1:5002/analyze + +# Scanning +REDACTOR_SCAN_PROFILE=file_scan +REDACTOR_SCAN_MAX_FILE_SIZE=10485760 +REDACTOR_SCAN_SKIP_BINARY=true +REDACTOR_SCAN_RESPECT_GITIGNORE=true +REDACTOR_SCAN_WINDOW_LINES=512 +REDACTOR_SCAN_OVERLAP_LINES=4 +REDACTOR_SCAN_DECODE=true +REDACTOR_SCAN_VERIFY=false +REDACTOR_SCAN_BASELINE=.redactor-baseline.json +``` + +Only the `default` profile reads the per-profile variables. The other shipped profiles set literal values, so `REDACTOR_MAX_VALUE_LENGTH` changes `default` and nothing else. + +## How Values Are Validated + +`env()` hands every value over as a string, so each key is coerced and checked when the profile is built: + +- Booleans accept `true`, `false`, `1`, `0`, and the strings `true`, `false`, `1`, `0`, `yes`, `no`, `on`, `off` and an empty string, case-insensitively. +- Integers must be positive; `max_value_length` and `max_object_size` also accept null, and an empty string from `env()` counts as null. +- `non_redactable_object_behavior`, `large_string_behavior` and a rule's `mode` and `validator` must be one of the documented values. +- `min_confidence` must be between 0 and 1. +- A pattern that does not compile is dropped from the profile; a rule with no `pattern` and no `words`, or with a bad `mode`, throws. + +Anything that fails throws a `ConfigurationException` whose message names the path, such as `profiles.default.max_depth`. `php artisan redactor:validate` surfaces all of them at once. + +## Region Packs + +`regions` at the top level holds pattern lists grouped by country: `gb`, `nl`, +`de`, `fr`, `it`, `es`, `be`, `se`, `no`, `ca`, `au` and `eu`. A profile's +`regions` key lists the packs to spread into its patterns. Packs are off unless +listed. See [Region Packs](rules.md#region-packs). + diff --git a/docs/entity-recognition.md b/docs/entity-recognition.md new file mode 100644 index 0000000..9be8099 --- /dev/null +++ b/docs/entity-recognition.md @@ -0,0 +1,166 @@ +# Entity Recognition + +- [Introduction](#introduction) +- [Enabling It](#enabling-it) +- [The Presidio Contract](#the-presidio-contract) +- [Gates](#gates) +- [Offsets](#offsets) +- [Batching](#batching) +- [The Circuit Breaker](#the-circuit-breaker) +- [Writing a Recognizer](#writing-a-recognizer) +- [When to Use It](#when-to-use-it) + +## Introduction + +Names, addresses and organisations are the PII no regex can express and no entropy measure can see. A named entity recogniser can find them, at a cost three orders of magnitude above the rule engine, so the package treats it as a gated extra rather than a default. + +`EntityRecognitionStrategy` is listed in the shipped `default` profile and does nothing until `recognition.enabled` is true. When it is disabled the strategy is left out of the chain entirely, so it costs nothing. + +## Enabling It + +```php +'recognition' => [ + 'enabled' => true, + 'driver' => 'presidio', + 'url' => 'http://presidio:5002/analyze', + 'language' => 'en', + 'entities' => ['PERSON', 'LOCATION', 'ORGANIZATION'], + 'entity_map' => ['PERSON' => 'person', 'LOCATION' => 'location', 'ORGANIZATION' => 'organization'], + 'score_threshold' => 0.6, +], + +'operators' => [ + 'person' => 'surrogate', + 'location' => 'redact', +], +``` + +Every key is described in [Configuration](configuration.md#recognition). A profile that wants recognition must list `EntityRecognitionStrategy` in its `strategies`; of the shipped profiles only `default` does. + +Recognised spans go through the same overlap resolution, confidence floor and operators as everything else. A `person` becomes a stable surrogate exactly the way an email does. Each finding has the rule name `entity_recognition` and, as its entity, whatever `entity_map` maps the recogniser's label to, or the label lowercased. Its confidence starts at the recogniser's own score and gains the [context boost](rules.md#confidence) beside a credential keyword. + +## The Presidio Contract + +The built-in driver speaks Presidio's `/analyze` contract: text in, a list of spans out. + +Request: + +```json +{ + "text": "Alice Smith lives in Berlin", + "language": "en", + "score_threshold": 0.6, + "entities": ["PERSON", "LOCATION"] +} +``` + +`entities` is omitted when the profile's list is empty, which asks for everything the recogniser knows. Response: + +```json +[ + {"entity_type": "PERSON", "start": 0, "end": 11, "score": 0.85}, + {"entity_type": "LOCATION", "start": 21, "end": 27, "score": 0.9} +] +``` + +Anything that speaks this works without a line of PHP: the reference Presidio analyzer, the same analyzer with a transformer recogniser, or a small wrapper around any fine-tuned model. Items with a missing or mistyped field are skipped. A non-2xx status or a non-list body counts as a failure. + +The driver posts with `timeout` seconds to wait and a connect timeout of at most one second. + +## Gates + +Only a value that passes every gate is sent: + +1. It is a string. +2. It is at least `min_length` and at most `max_length` bytes. +3. It has at least `min_words` whitespace-separated words. +4. It reads as prose: it does not start with `{`, `[` or `<`, and at least half its words are made of letters (with apostrophes, hyphens and ordinary punctuation allowed). + +A JSON blob, a stack trace or a bare token is not something a model reads well, and its guesses would be the false positives the gate exists to prevent. + +On the way back: + +5. Only spans scoring at or above `score_threshold` are kept. +6. Only labels in `entities` are kept, when the list is not empty. +7. Only spans whose offsets line up with the value are kept. See [Offsets](#offsets). + +## Offsets + +A recogniser reports character offsets, because every model tokenises its own copy of the text and none of them count bytes. The strategy converts each span's `start` to a byte offset with `mb_substr()`, extracts the text between `start` and `end`, and checks that the bytes at that position in the value are exactly that text. A span that is empty, whose `end` is not past its `start`, that runs past the end of the value, or that does not line up is skipped with a warning, never guessed. The detection's offset is then a byte offset like every other finding's. + +## Batching + +A model call costs a round trip and almost nothing per extra byte, so a record with fifty free-text fields should cost one call, not fifty. Before the walk starts, the strategy gathers every value that passes the gates, leaves out anything under a safe or blocked key, since the walk will preserve or replace those without reading them, and sends the rest in one request. Identical texts are sent once. When the walk later reaches a value, it finds the spans already recognised and asks nothing. + +The Presidio driver joins the texts with a blank line between them, sends them under one 60,000 character cap per request, and hands each span back to the text it fell in with offsets relative to that text. No entity spans a blank line, so a span that crosses the join is an artefact and is dropped. A recogniser of your own takes the list directly by implementing `BatchRecognizer`, and one that only implements `Recognizer` is asked once per text at the same point. + +A batch that fails counts once against the breaker and primes every gathered value with nothing, so the walk does not retry a dead recogniser once per value. A value the walk truncates before the strategy sees it falls back to a single call for that value. Set `batch` to `false` to always ask per value. + +## The Circuit Breaker + +A sidecar that is down fails every call at the full timeout, so inside a log tap every line would wait seconds to be told nothing. A recogniser that throws is skipped and the output is rules-only for that value. After `failure_threshold` consecutive failures the breaker opens for `cooldown` seconds and the recogniser is not asked again until it closes; one success closes it. + +Breaker state is per process and keyed by recogniser and profile. It is deliberately not shared through the cache, because a breaker that needed the cache to work would fail exactly when the cache does. Each failure and each unknown driver is logged through the package's re-entrancy guard, so the warning cannot loop back into the log tap that raised it. + +## Writing a Recognizer + +To plug in something that does not speak the Presidio contract, implement `Kirschbaum\Redactor\Recognition\Recognizer`: + +```php +use Kirschbaum\Redactor\Recognition\RecognizedSpan; +use Kirschbaum\Redactor\Recognition\Recognizer; + +class OnnxRecognizer implements Recognizer +{ + public function name(): string + { + return 'onnx'; + } + + /** + * @param array $entities the recogniser's own labels; empty means all + * @return array + */ + public function recognize(string $text, string $language, array $entities, float $scoreThreshold): array + { + $spans = []; + + foreach ($this->model->predict($text) as $prediction) { + $spans[] = new RecognizedSpan( + entity: $prediction->label, // 'PERSON' + start: $prediction->start, // character offset + end: $prediction->end, // character offset, exclusive + score: $prediction->score, // 0.0 to 1.0 + ); + } + + return $spans; + } +} +``` + +A recogniser that can read a list in one call also implements `Kirschbaum\Redactor\Recognition\BatchRecognizer`, whose `recognizeMany()` takes an array of texts and returns spans per input index, with offsets relative to each text. See [Batching](#batching). + +Register it, usually in a service provider's `boot()`, and select it by name: + +```php +Redactor::registerRecognizer(new OnnxRecognizer); +``` + +```php +'recognition' => ['enabled' => true, 'driver' => 'onnx', /* ... */], +``` + +A recogniser reports; it never rewrites. It may throw when it cannot answer, and the strategy turns that into "rules only, this time" and counts it against the breaker. Return character offsets, not bytes. + +## When to Use It + +Use it on profiles that run from queues, exports and scans, where a model call's milliseconds are lost in the job's seconds and the payload is prose: support transcripts, free-text notes, exported records. Pair it with `surrogate` for people so an exported conversation stays readable and consistent. + +Do not enable it on the request path or on a busy log channel. The rule engine costs microseconds per value; a recogniser costs milliseconds and a network round trip, per prose-shaped value, on every log line that carries one. The gates keep JSON and stack traces out, but a `message` field is prose by definition. + +Do not rely on it for credentials. A model finds names; a pattern finds keys. The rules run either way. + +The companion package `kirschbaum-development/redactor-onnx` removes the sidecar: the same gates, breaker and batching, with the model loaded once per worker. It suits queue workers, Octane and scans for the same reason a sidecar does, and a request that boots and exits for the same reason a sidecar does not. + +To run a model inside the PHP process instead of a sidecar, install [kirschbaum-development/redactor-onnx](https://github.com/kirschbaum-development/redactor-onnx), which registers an `onnx` driver over TransformersPHP. It implements `BatchRecognizer`, so batching applies unchanged. diff --git a/docs/extending.md b/docs/extending.md new file mode 100644 index 0000000..00b5dc4 --- /dev/null +++ b/docs/extending.md @@ -0,0 +1,441 @@ +# Extending + +- [Introduction](#introduction) +- [Contracts at a Glance](#contracts-at-a-glance) +- [Strategies](#strategies) + - [The Strategy Contract](#the-strategy-contract) + - [The Marker Interfaces](#the-marker-interfaces) + - [A Worked Custom Strategy](#a-worked-custom-strategy) + - [Registering a Strategy](#registering-a-strategy) + - [What a Strategy Can Reach](#what-a-strategy-can-reach) +- [Detectors](#detectors) +- [Operators](#operators) + - [A Worked Custom Operator](#a-worked-custom-operator) +- [Surrogate Generators](#surrogate-generators) +- [Recognizers](#recognizers) +- [Verifiers](#verifiers) +- [Token Stores](#token-stores) +- [Macros](#macros) + +## Introduction + +The package has one seam for each thing it does: a strategy for a step in the chain, a detector for finding spans, an operator for replacing them, a generator for shaping a surrogate, a recogniser for asking a model, a verifier for asking a provider, and a token store for keeping originals. Each is an interface under `Kirschbaum\Redactor`, and each is registered in one place. + +## Contracts at a Glance + +| Contract | Answers | Registered with | +| --- | --- | --- | +| `Strategies\Contracts\Strategy` | Should I handle this value, and what replaces it? | `custom_strategies` config, or `Redactor::registerCustomStrategy()` | +| `Detection\Detector` | Where in this string is something sensitive? | Implement it on a strategy | +| `Operators\Operator` | What replaces a detected span? | `Redactor::registerOperator()` | +| `Operators\Surrogates\SurrogateGenerator` | What does a fake of this value look like? | `SurrogateFactory::register()` | +| `Recognition\Recognizer` | Where are the names and places in this prose? | `Redactor::registerRecognizer()` | +| `Verification\Verifier` | Is this credential still live? | The `SecretVerifier` constructor | +| `Tokenization\TokenStore` | Where does a token's original live? | Bind in the container | + +## Strategies + +### The Strategy Contract + +A strategy is one step of the chain a value passes through: + +```php +namespace Kirschbaum\Redactor\Strategies\Contracts; + +use Kirschbaum\Redactor\RedactionContext; + +interface Strategy +{ + public function shouldHandle(mixed $value, string $key, RedactionContext $context): bool; + + public function handle(mixed $value, string $key, RedactionContext $context): mixed; +} +``` + +`shouldHandle()` is asked for every value, scalar and container alike, with the key it sits under (`''` for a bare string or the root). When it returns true, `handle()` returns what should stand in the value's place. By default the chain stops there and the walk does not descend, so a strategy that returns a container has ended the matter for everything inside it. + +### The Marker Interfaces + +Four empty interfaces in the same namespace change how the chain treats a strategy: + +| Marker | Effect | +| --- | --- | +| `ChainableStrategy` | `handle()` rewrote part of the value rather than replacing it, so the strategies after it still run on what it returned. `LargeStringStrategy` is one. | +| `DetectingStrategy` | Extends `ChainableStrategy`. `handle()` returns the value untouched and reports what it found through `$context->collect()`. Every detecting strategy sees the same original string, and the context rewrites it once after the last of them. `RegexPatternsStrategy`, `ShannonEntropyStrategy`, `KnownSecretsStrategy` and `EntityRecognitionStrategy` are these. | +| `PreservingStrategy` | `handle()` declares the value safe. The chain ends, the walk does not descend, and any pending detections are discarded. `SafeKeysStrategy` is one. | +| `ConditionalStrategy` | Adds `appliesTo(RedactorConfig $config): bool`. A strategy returning false is left out of the chain for that profile, so it costs nothing. `EntityRecognitionStrategy` uses it to stay inert until enabled. | +| `PrimingStrategy` | Adds `prime(mixed $content, RedactionContext $context): void`, called once with the whole payload before the walk starts. For work that costs per call rather than per value: do it here in one go and leave the result on the context for `handle()` to read. `EntityRecognitionStrategy` uses it to recognise every prose value in one request. | + +Implement the markers that describe what your `handle()` does. A strategy with none of them replaces the value outright and ends the chain. + +### A Worked Custom Strategy + +A strategy that redacts anything under a key that starts with `internal_` or `debug_`: + +```php +namespace App\Redaction; + +use Kirschbaum\Redactor\RedactionContext; +use Kirschbaum\Redactor\Strategies\Contracts\Strategy; + +class InternalDataStrategy implements Strategy +{ + public function shouldHandle(mixed $value, string $key, RedactionContext $context): bool + { + return str_starts_with($key, 'internal_') || str_starts_with($key, 'debug_'); + } + + public function handle(mixed $value, string $key, RedactionContext $context): mixed + { + // recordRedaction() sets the flag, adds the key to redactedKeys, and + // reports a finding under the given rule name. markRedacted() would + // set the flag alone. + $context->recordRedaction($key, 'internal_data'); + + return '[INTERNAL]'; + } +} +``` + +A strategy that wants the operator policy, so the profile decides what happens, builds a `Detection` and hands it to the context instead of choosing a replacement itself: + +```php +use Kirschbaum\Redactor\Detection\Confidence; +use Kirschbaum\Redactor\Detection\Detection; + +public function handle(mixed $value, string $key, RedactionContext $context): mixed +{ + if (! is_string($value) || $context->isAllowed($value)) { + return $value; + } + + $detection = new Detection( + entity: 'internal', + rule: 'internal_data', + offset: 0, + value: $value, + confidence: Confidence::of(Confidence::CERTAIN, 'the key names internal data'), + key: $key, + ); + + $context->recordDetection($detection); + + return $context->operate($detection); +} +``` + +`operate()` resolves the operator through the same precedence as every shipped strategy and never throws. + +### Registering a Strategy + +Register the class under a short name in `custom_strategies`, then list the name in a profile's `strategies` wherever it should run: + +```php +'custom_strategies' => [ + 'internal_data' => \App\Redaction\InternalDataStrategy::class, +], + +'profiles' => [ + 'default' => [ + 'strategies' => [ + \Kirschbaum\Redactor\Strategies\SafeKeysStrategy::class, + 'internal_data', + \Kirschbaum\Redactor\Strategies\BlockedKeysStrategy::class, + // ... + ], + ], +], +``` + +Or register an instance at runtime, which also lets you pass constructor arguments: + +```php +Redactor::registerCustomStrategy('internal_data', new InternalDataStrategy($settings)); +``` + +A name registered at runtime takes precedence over the config entry of the same name, and registering one drops the cached strategy chains so the next redaction picks it up. Classes are instantiated without arguments from config, so a strategy that needs dependencies is registered at runtime. Custom strategies are cloned into each chain. + +A profile may also list a fully qualified class name directly, without registering it, as the shipped profiles do. + +### What a Strategy Can Reach + +`RedactionContext` is what a strategy is handed. The parts meant for strategies: + +| Member | Use | +| --- | --- | +| `$context->config` | The resolved `RedactorConfig`: `replacement`, `patterns`, `minConfidence`, `shannonEntropy`, `recognition` and so on. | +| `$context->operate(Detection $detection, ?OperatorSpec $atLocation = null)` | Apply the configured operator to a detection and return the replacement text. | +| `$context->operatorSpecFor(Detection $detection)` | Which operator would apply, without applying it. | +| `$context->collect(Detection $detection)` | Hold a detection for resolution, from a `DetectingStrategy`. | +| `$context->isAllowed(string $value)` | Whether the profile allow-list excuses the value. | +| `$context->recordRedaction(string $key, ?string $rule, int $offset, int $length, string $matched, ?string $entity, ?Confidence $confidence, bool $redacted)` | Record that something was redacted under a key and report a finding. | +| `$context->recordDetection(Detection $detection, bool $redacted = true)` | The same, from a detection. | +| `$context->markRedacted()` | Set the redaction flag alone. | +| `$context->secrets()` | The known secrets in play, profile and runtime. | +| `$context->pseudonymizer()` | The profile's pseudonymizer, or null. | +| `$context->recognizers()` | The recogniser registry. | +| `$context->getCachedEntropy()`, `cacheEntropy()` | The per-redaction entropy cache. | + +## Detectors + +`Detection\Detector` is the contract for anything that can find sensitive spans in a string: + +```php +namespace Kirschbaum\Redactor\Detection; + +use Kirschbaum\Redactor\RedactionContext; + +interface Detector +{ + /** @return array offsets relative to $subject as given */ + public function detect(string $subject, string $key, RedactionContext $context): array; +} +``` + +A detector reads; it never writes. It reports every span it believes is sensitive with an entity, a rule name, a byte offset into the subject exactly as received, the matched text and a `Confidence`, and leaves the decisions to the context. The shipped detecting strategies implement both `Strategy` and `Detector`, with `handle()` collecting whatever `detect()` returns: + +```php +class MyDetector implements DetectingStrategy, Detector, Strategy +{ + public function shouldHandle(mixed $value, string $key, RedactionContext $context): bool + { + return is_string($value); + } + + public function handle(mixed $value, string $key, RedactionContext $context): mixed + { + foreach ($this->detect($value, $key, $context) as $detection) { + $context->collect($detection); + } + + return $value; + } + + public function detect(string $subject, string $key, RedactionContext $context): array + { + // ... + } +} +``` + +Build a `Confidence` with `Confidence::of($score, $reason)` and add signals with `->with($name, $delta, $reason)`. `KeywordContext::boost($confidence, $subject, $offset, $key)` adds the shared context signal. Where the detector fails and the value cannot be trusted, return `[Detection::failClosed($entity, $rule, $subject, $key, $reason)]`, which replaces the whole value whatever the operator policy says. + +## Operators + +An operator produces the text that replaces a detected span: + +```php +namespace Kirschbaum\Redactor\Operators; + +use Kirschbaum\Redactor\Detection\Detection; + +interface Operator +{ + public function apply(Detection $detection, OperatorContext $context): string; +} +``` + +Whether anything changed is decided by comparing the returned text with the detected value: an operator that returns `$detection->value` unchanged, as `preserve` does, has its finding reported without the payload being marked redacted. `OperatorContext` is deliberately narrow: an operator receives the profile's `replacement`, its own `options` and a lazily resolved pseudonymizer, and nothing else. It cannot reach the payload, the profile or the container, which keeps it testable in isolation and impossible to turn into a second detection layer. + +### A Worked Custom Operator + +An operator that replaces a value with its category, so a log says `` rather than `[REDACTED]`: + +```php +namespace App\Redaction; + +use Kirschbaum\Redactor\Detection\Detection; +use Kirschbaum\Redactor\Operators\Operator; +use Kirschbaum\Redactor\Operators\OperatorContext; + +class ClassifyOperator implements Operator +{ + public function apply(Detection $detection, OperatorContext $context): string + { + $open = $context->stringOption('open', '<'); + $close = $context->stringOption('close', '>'); + + return $open.$detection->entity.$close; + } +} +``` + +Register it, in a service provider's `boot()`, and use it from config by name: + +```php +Redactor::registerOperator('classify', new ClassifyOperator); +``` + +```php +'operators' => [ + 'default' => 'classify', + 'phone' => ['classify' => ['open' => '[', 'close' => ']']], +], +``` + +`OperatorContext` exposes the profile's `replacement` and the raw `options` array as public properties, offers `intOption()`, `boolOption()` and `stringOption()`, each returning the default when the option is absent or the wrong type, and `pseudonymizer()` for an operator that needs a stable mapping. An operator that pseudonymises should return `$context->replacement` when `pseudonymizer()` is null, as the shipped ones do, rather than emit an unkeyed stand-in. + +Registering a name that already exists replaces the built-in operator, so an application can, for example, swap `tokenize` for one backed by a vault. + +## Surrogate Generators + +The `surrogate` operator asks a `SurrogateFactory` for the first generator that supports the value. Add one for a domain type the package has never heard of, a policy number, an NHS number, an internal account format: + +```php +namespace App\Redaction; + +use Kirschbaum\Redactor\Operators\Surrogates\SurrogateGenerator; +use Kirschbaum\Redactor\Support\DeterministicRandom; + +class PolicyNumberSurrogate implements SurrogateGenerator +{ + public function supports(string $entity, string $value): bool + { + return $entity === 'policy_number'; + } + + public function generate(string $value, DeterministicRandom $random, array $options = []): string + { + return 'POL-'.$random->token(8, 'ABCDEFGHJKLMNPQRSTUVWXYZ23456789'); + } +} +``` + +`DeterministicRandom` is a keyed, reproducible stream seeded from the value, so the same input yields the same surrogate on any machine. It offers `byte()`, `below($bound)`, `pick($alphabet)`, `digit()` and `token($length, $alphabet)`. + +Generators are registered ahead of the built-in ones on the factory, and the factory is given to the `SurrogateOperator`, which is registered under `surrogate`: + +```php +use Kirschbaum\Redactor\Operators\SurrogateOperator; +use Kirschbaum\Redactor\Operators\Surrogates\SurrogateFactory; + +$factory = new SurrogateFactory([new PolicyNumberSurrogate]); + +Redactor::registerOperator('surrogate', new SurrogateOperator($factory)); +``` + +`CharacterClassSurrogate` supports everything and stays last, so a generator that claims a value wins over it. + +## Recognizers + +Implement `Recognition\Recognizer` and register it with `Redactor::registerRecognizer()`. [Entity Recognition](entity-recognition.md#writing-a-recognizer) has the full contract and a worked example. + +## Verifiers + +A verifier asks one provider whether one credential is live: + +```php +namespace Kirschbaum\Redactor\Verification; + +interface Verifier +{ + public function name(): string; + + public function host(): string; + + public function supports(string $entity, string $rule): bool; + + public function verify(string $secret): VerificationResult; +} +``` + +`name()` is what goes in `scan.verification.verifiers`. `host()` is announced before any request is sent. `verify()` must never throw or log the secret, and returns `VerificationResult::active()`, `::inactive()` or `::unknown()`, each with an optional note: + +```php +namespace App\Redaction; + +use Illuminate\Support\Facades\Http; +use Kirschbaum\Redactor\Verification\VerificationResult; +use Kirschbaum\Redactor\Verification\Verifier; +use Throwable; + +class TwilioKeyVerifier implements Verifier +{ + public function name(): string { return 'twilio_key'; } + + public function host(): string { return 'api.twilio.com'; } + + public function supports(string $entity, string $rule): bool + { + return $entity === 'twilio_key'; + } + + public function verify(string $secret): VerificationResult + { + try { + $response = Http::withToken($secret)->timeout(5)->get('https://api.twilio.com/2010-04-01/Accounts.json'); + + return match (true) { + $response->status() === 401 => VerificationResult::inactive('Twilio rejected the key (401).'), + $response->successful() => VerificationResult::active('Twilio accepted the key; rotate it.'), + default => VerificationResult::unknown(sprintf('Twilio returned %d.', $response->status())), + }; + } catch (Throwable $e) { + return VerificationResult::unknown('Could not reach Twilio: '.$e->getMessage()); + } + } +} +``` + +The shipped verifiers are hard-wired into `SecretVerifier`; there is no registry for verifiers yet. To use your own, construct a `SecretVerifier` with the full list and give it to the scanner: + +```php +use Kirschbaum\Redactor\Scanner\Scanner; +use Kirschbaum\Redactor\Verification\SecretVerifier; + +$verifier = new SecretVerifier( + allowed: ['twilio_key', 'github_token'], + verifiers: [new TwilioKeyVerifier, new GitHubTokenVerifier], +); + +$result = app(Scanner::class)->withVerifier($verifier)->scanFile($path, 'file_scan'); +``` + +`SecretVerifier::fromConfig($settings, $verifiers)` applies the same three-gate rule to a custom list. + +## Token Stores + +`Tokenization\TokenStore` is where a token's original lives while the token is out in the world: + +```php +namespace Kirschbaum\Redactor\Tokenization; + +interface TokenStore +{ + public function put(string $token, string $value, string $entity, ?int $ttlSeconds = null): void; + + public function get(string $token): ?string; + + public function forget(string $token): void; +} +``` + +`get()` returns null for a token it does not know, and the detokenizer leaves such tokens alone. The shipped `CacheTokenStore` encrypts each value with the application's encrypter before writing it to the cache. + +To use another store, bind it in a service provider's `register()`. The package resolves the store lazily the first time a `tokenize` operator runs, so the binding is picked up as long as it exists before then: + +```php +use Kirschbaum\Redactor\Tokenization\TokenStore; + +$this->app->singleton(TokenStore::class, fn () => new VaultTokenStore($this->app->make(Vault::class))); +``` + +## Macros + +Both `Kirschbaum\Redactor\Redactor` and `Kirschbaum\Redactor\PendingRedaction` are `Macroable`, so an application can add its own methods to the facade and to the fluent builder: + +```php +use Kirschbaum\Redactor\PendingRedaction; +use Kirschbaum\Redactor\Redactor; + +Redactor::macro('forExport', fn (mixed $content) => $this->profile('observability')->withoutMarkers()->redact($content)); + +PendingRedaction::macro('strictly', fn () => $this->profile('strict')); +``` + +```php +Redactor::forExport($rows); +Redactor::profile(null)->strictly()->redact($data); +``` + +Both are also `Conditionable`, so `when()` and `unless()` are available on the redactor and on a pending redaction. diff --git a/docs/getting-started.md b/docs/getting-started.md new file mode 100644 index 0000000..d3f954f --- /dev/null +++ b/docs/getting-started.md @@ -0,0 +1,246 @@ +# Getting Started + +- [Introduction](#introduction) +- [Installation](#installation) +- [Redacting Logs](#redacting-logs) +- [Your First Redaction](#your-first-redaction) +- [Inspecting a Redaction](#inspecting-a-redaction) +- [The Fluent Builder](#the-fluent-builder) +- [Profiles](#profiles) + - [The Shipped Profiles](#the-shipped-profiles) + - [Strategies](#strategies) +- [Exceptions](#exceptions) +- [Validating Your Configuration](#validating-your-configuration) +- [Next Steps](#next-steps) + +## Introduction + +Redactor takes a value, a string, an array, an object, and returns a copy with the sensitive parts replaced. It finds them three ways: by the name of the key a value sits under, by what the value looks like, and by where it lives in the payload. What replaces a detected value is decided separately, so the same email address can become `[REDACTED]` in one log channel and a stable pseudonym in another. + +The package registers a service provider and a facade automatically. Nothing else runs until you ask it to. + +## Installation + +Install the package with Composer: + +```bash +composer require kirschbaum-development/redactor +``` + +Then publish the configuration file: + +```bash +php artisan vendor:publish --tag=redactor-config +``` + +This writes `config/redactor.php`. You can run the package without publishing it; the defaults described in [Configuration](configuration.md) apply. Publish it when you want to add your own rules or profiles. + +Redactor requires PHP 8.3, 8.4 or 8.5 and Laravel 12 or 13. + +## Redacting Logs + +The most common boundary is the log. Add the tap to any channel in `config/logging.php`: + +```php +'channels' => [ + 'stack' => [ + 'driver' => 'stack', + 'channels' => explode(',', env('LOG_STACK', 'single')), + 'ignore_exceptions' => false, + 'tap' => [Kirschbaum\Redactor\Logging\RedactorTap::class], + ], +], +``` + +`RedactorTap` pushes a Monolog processor onto the channel. The processor redacts each record's message, context and extra, then leaves the channel's own formatter alone, so a channel that writes JSON keeps writing JSON. + +Append a profile name after a colon when a channel needs something other than the default profile: + +```php +'audit' => [ + 'driver' => 'daily', + 'path' => storage_path('logs/audit.log'), + 'tap' => [Kirschbaum\Redactor\Logging\RedactorTap::class.':strict'], +], +``` + +Redaction in the log path never throws. If a profile is misconfigured or a strategy fails on unexpected input, the record's content is replaced rather than emitted and logging carries on. See [Boundaries](boundaries.md#log-channels) for the formatter alternative and what happens to exceptions and other opaque objects. + +## Your First Redaction + +Call `redact()` on the facade with anything you are about to emit: + +```php +use Kirschbaum\Redactor\Facades\Redactor; + +$redacted = Redactor::redact([ + 'user_id' => 123, + 'password' => 'secret123', + 'api_key' => 'sk-1234567890abcdef1234567890abcdef12345678', + 'email' => 'user@example.com', +]); + +// [ +// 'user_id' => 123, // safe key: preserved +// 'password' => '[REDACTED]', // blocked key +// 'api_key' => '[REDACTED]', // blocked key (*key*) +// 'email' => '[REDACTED]', // blocked key +// '_redacted' => true, // marker, on by default +// ] +``` + +Inside a string, only the sensitive span is replaced. The text around it survives: + +```php +Redactor::redact('User bob@example.com placed order 123'); + +// 'User [REDACTED] placed order 123' +``` + +Pass a profile name as the second argument to use something other than the default profile: + +```php +Redactor::redact($data, 'strict'); +``` + +Objects are handled too. A Laravel model or anything with a `toArray()` method is walked through that; other objects are walked through their JSON form. Throwables, `DateTimeInterface`, `DateTimeZone`, enums and closures pass through untouched, since taking them apart would destroy them, though key rules still apply to the key they sit under. + +## Inspecting a Redaction + +When you need to know whether anything matched, call `inspect()` instead of reading the `_redacted` marker back out of the payload: + +```php +$result = Redactor::inspect($data); + +$result->value; // the redacted payload +$result->wasRedacted; // bool +$result->redactedKeys; // ['password', 'api_key', 'email'] +$result->findings; // an array of MatchFinding +``` + +Each `MatchFinding` carries the rule that fired, the entity it found, the key, a byte offset and length into the value you passed, and a confidence score with the signals behind it: + +```php +foreach ($result->findings as $finding) { + $finding->rule; // 'email' + $finding->entity(); // 'email' + $finding->key; // 'email' + $finding->offset; // 0 + $finding->length; // 16 + $finding->confidence; // a Confidence, or null for a blocked key +} +``` + +`RedactionResult` and `MatchFinding` are both `Arrayable` and `JsonSerializable`. The array form of a finding omits the matched text, so a result can be logged without becoming the leak it reports. + +`inspect()` also takes a third argument, `$mark`, which overrides the profile's `mark_redacted` setting for one call: + +```php +Redactor::inspect($data, 'default', mark: false)->value; +``` + +## The Fluent Builder + +`Redactor::profile()` returns a `PendingRedaction` you can configure before running it: + +```php +Redactor::profile('strict')->redact($data); + +Redactor::profile('observability')->withoutMarkers()->inspect($data); + +Redactor::profile('audit') + ->when($verbose, fn ($redaction) => $redaction->withMarkers()) + ->redact($data); + +Redactor::profile('default')->only(['email', 'credit_card'])->redact($export); +``` + +The builder offers: + +| Method | Effect | +| --- | --- | +| `profile(?string $profile)` | Use the given profile, or `null` for the default. | +| `withMarkers()` | Write the `_redacted` markers into the payload, whatever the profile says. | +| `withoutMarkers()` | Never write the markers. | +| `only(array $entities)` | Act only on these entities this time. A key rule's entity is the key name; a path rule's is the key it lands on. | +| `except(array $entities)` | Act on every entity but these. Composes with `only()`. | +| `redact(mixed $content)` | Redact and return the value. | +| `inspect(mixed $content)` | Redact and return a `RedactionResult`. | +| `redactSafely(mixed $content)` | Redact without ever throwing. | + +`only()` and `except()` never suppress a fail-closed detection: a value a +pattern could not evaluate is still replaced whatever the filter says. + +Both the redactor and the pending redaction are `Conditionable` and `Macroable`, so `when()` and `unless()` work as they do elsewhere in Laravel, and you can add your own methods. See [Extending](extending.md#macros). + +## Profiles + +A profile is one complete redaction configuration: which strategies run and in what order, which keys are safe or blocked, which patterns to look for, and what to do with what is found. Profiles live under `profiles` in `config/redactor.php`, and every entry point accepts a profile name. + +Profiles exist because the right amount of redaction depends on where the data is going. An audit log wants everything gone; an application log wants values pseudonymised so you can still count users; a file scan wants content rules only, since there are no keys to match. + +You can list and check profiles at runtime: + +```php +Redactor::profiles(); // ['default', 'strict', 'file_scan', 'observability', 'performance'] +Redactor::hasProfile('audit'); // false +Redactor::strategies('strict'); // the resolved strategy chain +``` + +### The Shipped Profiles + +| Profile | Intended for | What makes it different | +| --- | --- | --- | +| `default` | Application logs and general use | Every strategy, the shared credential and identity rules, entity recognition present but inert, cards partially masked. | +| `strict` | Audit trails and sensitive contexts | More blocked keys (`phone`, `address`, `ip`, `name`, `username`), IPv4 and UUID patterns, entropy threshold 4.0 over 15 characters, `max_value_length` 1000, non-redactable objects redacted. | +| `observability` | Logs you still need to reason about | Pseudonymises emails, phones, IPs and cards with stable surrogates instead of redacting them; `min_confidence` 0.4; no markers. | +| `file_scan` | `redactor:scan` | No key strategies at all; adds labelled rules such as `api_key_generic`, `aws_secret_key`, `base64_key` and `password_assignment`. | +| `performance` | High-throughput paths | Key rules, known secrets and two patterns (`email`, `simple_token`); no size limits, no entropy, no markers. | + +The `default`, `strict`, `observability` and `file_scan` profiles share the same credential and identity rules. They are defined once at the top of the config file and spread into each profile, so a credential one profile catches is caught by the others. [Configuration](configuration.md#the-shipped-profiles-compared) compares every setting side by side. + +### Strategies + +A profile lists its strategies in the order they run. The shipped ones are: + +| Strategy | Role | +| --- | --- | +| `SafeKeysStrategy` | Preserves values under safe keys, subtree included, and stops the walk. | +| `BlockedKeysStrategy` | Redacts values under blocked keys. | +| `LargeObjectStrategy` | Replaces arrays and objects with more items than `max_object_size`. | +| `LargeStringStrategy` | Truncates strings over `max_value_length`, keeping and scanning the head. | +| `KnownSecretsStrategy` | Finds the application's own credentials wherever they appear verbatim. | +| `RegexPatternsStrategy` | Finds spans by pattern. | +| `ShannonEntropyStrategy` | Finds tokens random enough to be a credential. | +| `EntityRecognitionStrategy` | Asks a named entity recogniser about free text. Inert until `recognition.enabled` is true. | + +The chain stops at the first strategy that replaces a value outright. The last four are *detectors*: they report what they found and where, change nothing, and once every detector has seen the value the context resolves the reports and rewrites the string once. [Rules](rules.md#how-detections-are-resolved) explains what follows from that. + +## Exceptions + +Everything the package throws implements `Kirschbaum\Redactor\Exceptions\RedactorException`, so one `catch` covers all of it: + +| Exception | Extends | Thrown when | +| --- | --- | --- | +| `ConfigurationException` | `InvalidArgumentException` | A profile or rule is misconfigured. The message names the config path. | +| `ProfileNotFoundException` | `ConfigurationException` | The requested profile is not configured. | +| `PseudonymizationKeyException` | `RuntimeException` | No key strong enough to pseudonymise with could be produced. | +| `GitException` | `RuntimeException` | The scanner asked git and git could not answer. | + +`redact()` and `inspect()` throw. `redactSafely()` does not: it catches everything, logs a warning through the package's re-entrancy guard, and returns the profile's replacement string followed by ` (redaction failed)`. That is the method the log processor, the middleware and the stream redactor use, since a throw inside the pipeline that reports errors would take the error with it. + +## Validating Your Configuration + +A broken profile throws the first time something uses it, which in the log path means it is replaced rather than emitted. Find out at deploy time instead: + +```bash +php artisan redactor:validate +``` + +The command resolves every profile, builds its strategy chain, checks that no key is listed as both safe and blocked, and runs every rule's samples through the real detection path. It exits non-zero if anything fails. See [Testing](testing.md#validating-profiles). + +## Next Steps + +- [Boundaries](boundaries.md) covers every place data leaves an application and the adapter for each. +- [Rules](rules.md) explains how to write patterns that catch what you mean and nothing else. +- [Operators and Pseudonymisation](operators-and-pseudonymisation.md) shows how to keep logs joinable after redaction. diff --git a/docs/operators-and-pseudonymisation.md b/docs/operators-and-pseudonymisation.md new file mode 100644 index 0000000..1181b1e --- /dev/null +++ b/docs/operators-and-pseudonymisation.md @@ -0,0 +1,202 @@ +# Operators and Pseudonymisation + +- [Introduction](#introduction) +- [Configuring Operators](#configuring-operators) +- [The Operators](#the-operators) +- [Precedence](#precedence) +- [Pseudonymisation](#pseudonymisation) + - [Surrogates](#surrogates) + - [The Key and the Salt](#the-key-and-the-salt) + - [Cross-Profile Stability](#cross-profile-stability) +- [Reversible Tokens](#reversible-tokens) + - [Detokenizing](#detokenizing) + - [The Token Store](#the-token-store) +- [Trust Model](#trust-model) + +## Introduction + +Detection asks "is this sensitive". An operator answers "so what". They are separate because the right answer differs by context for the very same value: an email in an audit log wants a stable surrogate so the log stays joinable, the same email in a support export wants deleting, and in a secret scan it wants reporting and nothing else. + +Operators are chosen per *entity*, the kind of thing that was found, rather than per rule. Every detector goes through the same policy: a value found by its key uses the lowercased key name as its entity, a high-entropy token has the entity `high_entropy`, a known secret has `known_secret`, a pattern match has whatever its rule's `entity` says. + +## Configuring Operators + +A profile's `operators` block maps entities to operators, with `default` for everything else: + +```php +'operators' => [ + 'default' => 'redact', + 'email' => ['surrogate' => ['preserve_domain' => true]], + 'credit_card' => ['partial' => ['keep' => 4]], + 'ssn' => 'nullify', +], +``` + +An operator is written in any of three spellings, depending on how much you are saying: + +```php +'redact' // just the name +['partial' => ['keep' => 4]] // the name mapped to its options +['operator' => 'partial', 'keep' => 4] // the name as a key beside its options +``` + +The same spellings work in a pattern rule's `operator` option and as the value of a path rule. An operator name that is not registered is not caught when the profile is built: at redaction time the value is redacted with the replacement string and a warning names the operator, the profile and the rule. + +## The Operators + +| Operator | `alice@customer.com` becomes | Options | +| --- | --- | --- | +| `redact` | `[REDACTED]` | none; writes the profile's `replacement` | +| `mask` | `******************`, length preserved | `mask_character` (default `*`) | +| `partial` | `**************.com`, last N kept | `keep` (default 4), `mask_character` | +| `remove` | deleted | none | +| `nullify` | `null` in a field; deleted inside a string | none | +| `hash` | `[email:k4m9rp2xzq]` | `length` (4 to 64, default 10), `labelled` (default true) | +| `surrogate` | `u_7f3ac9@customer.com` | depends on the generator, see [Surrogates](#surrogates) | +| `tokenize` | `tok_email_k4m9rp2xzq` | `ttl` in seconds | +| `preserve` | `alice@customer.com`, reported and unchanged | none | + +A few of them deserve a note: + +**`nullify`** exists for typed fields. `[REDACTED]` in an integer field breaks every consumer that typed it, and `remove` breaks the ones that require the key. Null keeps both honest: the field is there, it has no value. Where a consumer has typed the field, an API contract or MCP structured content, use `nullify`. Inside a string there is no null to write, so a span found by a pattern is deleted as `remove` would. + +**`hash`** produces a stable keyed token that is obviously not real data, for places where a format-preserving surrogate might be mistaken for the genuine value. With `labelled` false it is the bare token. + +**`preserve`** is not a no-op. It lets a scan profile detect and report without rewriting anything, and lets one path rule carve an exception out of a broader one. A preserved finding appears in `inspect()` without marking the payload redacted. + +`hash`, `surrogate` and `tokenize` need a pseudonymisation key. Without one they fall back to `redact` rather than emit an unkeyed stand-in that would look joinable and silently not be. + +Register your own with `Redactor::registerOperator('classify', $operator)` and use it from config by name. See [Extending](extending.md#operators). + +## Precedence + +When a detection has more than one candidate operator, the most specific wins: + +1. The **path** it was found at, when a path rule matched. +2. The **entity** it is: `operators.`. +3. The **rule** that found it, when the rule explicitly set `operator` or a non-default `mode`. +4. The profile **default**: `operators.default`, or `redact` when that is absent. + +Entity beats rule deliberately. "Every email here becomes a surrogate" is a policy decision about data, and which regex spotted it is an implementation detail. Rule beats default only when the rule actually chose something: a rule's `mode` defaults to `replace`, and treating that default as a choice would make `operators.default` unreachable for anything found by a pattern. + +## Pseudonymisation + +Replacing every value with `[REDACTED]` collapses distinct values into one, which destroys the questions logs exist to answer: how many users hit this, is it always the same account, did this session span both services. + +`surrogate`, `hash` and `tokenize` replace a value with a *stable* stand-in instead. The same input always produces the same output, so counts, joins and traces survive: + +```php +Redactor::redact('login by alice@customer.com', 'observability'); +// 'login by u_7f3ac9@customer.com' + +Redactor::redact('logout for alice@customer.com', 'observability'); +// 'logout for u_7f3ac9@customer.com' same surrogate, still joinable +``` + +Inputs are normalised before they are keyed, so `Bob@Example.COM ` and `bob@example.com` produce the same surrogate rather than double-counting one user. + +### Surrogates + +A surrogate preserves the shape of the value it replaces, so anything downstream that parses the value keeps parsing it. The `surrogate` operator picks the first generator that supports the value: + +| Generator | Supports | Example | Options | +| --- | --- | --- | --- | +| `EmailSurrogate` | entity `email`, or a value with exactly one `@` | `alice@customer.com` to `u_7f3ac9@customer.com` | `preserve_domain` (default true). When false the domain becomes `example.invalid`, which can never resolve. | +| `CreditCardSurrogate` | entity `credit_card`, or 12 to 19 digits | `4111 1111 1111 1111` to `4111 1193 7420 8846`, Luhn-valid, spacing kept | `preserve_bin` (default 6). Digits of the issuer prefix to keep. | +| `CharacterClassSurrogate` | everything | `sk_live_4eC39HqLyj` to `sk_live_9mB71TzKnQ`; `+1 (555) 867-5309` to `+7 (204) 331-8874` | `preserve_prefix` (default 0). Leading bytes to keep verbatim. | + +`CharacterClassSurrogate` replaces each letter and digit with another of the same class and leaves separators, punctuation and multibyte characters alone. Length, capitalisation and digit positions survive; nothing of the original does except its shape. It is the fallback for every entity nobody wrote a generator for. To add one, see [Extending](extending.md#surrogate-generators). + +Keeping an email's domain preserves the analysis people run on logs: which tenant, which provider, how many distinct users at one company. Keeping a card's BIN preserves the issuer and card type, which fraud and finance teams aggregate on and which is not specific to a cardholder. + +### The Key and the Salt + +The mapping is one-way: an HMAC, not encryption. There is no route from a surrogate back to the original, but anyone holding the key can confirm a guess, so **the key must not travel with the logs**. + +```php +'pseudonymization' => [ + 'enabled' => env('REDACTOR_PSEUDONYMIZATION', true), + 'key' => env('REDACTOR_PSEUDONYMIZATION_KEY'), + 'salt' => env('REDACTOR_PSEUDONYMIZATION_SALT'), +], +``` + +Leave `key` null to derive one from `APP_KEY`. The derivation is an HMAC over a fixed label, so `APP_KEY` itself is never used directly and a leaked surrogate corpus cannot be turned against anything else signed with it. An explicit key must be at least 16 bytes; a shorter one throws a `PseudonymizationKeyException`, which the operators catch and log before falling back to plain redaction. + +Rotating the key changes every surrogate. That is the intended way to break correlation with logs already exported, and the reason not to rotate it casually. + +Without a usable key, because pseudonymisation is disabled, `APP_KEY` is empty, or the key is too short, `surrogate`, `hash` and `tokenize` fall back to plain redaction and a warning is logged once per redaction. + +### Cross-Profile Stability + +The salt is shared by every profile, so the same user gets the same surrogate on every channel and an audit log on `strict` can be joined with an application log on `observability`. The entity is part of the seed, so the same string found as an `email` and as a `phone` gets different surrogates. + +A profile that must not be linkable back sets its own salt: + +```php +'export' => [ + 'pseudonymization' => ['salt' => 'export-2026'], + // ... +], +``` + +A profile can also set `'pseudonymization' => ['enabled' => false]` to fall back to plain redaction for that profile alone. + +## Reversible Tokens + +A surrogate is one-way. A token is a surrogate the *application* can exchange back, which is what the boundary in front of a language model needs: the model sees `tok_email_k4m9rp2xzq`, refers to it in its answer, and the application resolves it before acting. + +```php +'operators' => [ + 'email' => 'tokenize', + 'credit_card' => ['tokenize' => ['ttl' => 600]], +], +``` + +```php +$prompt = Redactor::redact($ticket, 'ai'); // 'reply to tok_email_k4m9rp2xzq about ...' +$answer = $llm->complete($prompt); // the model reasons about the token +$action = Redactor::detokenize($answer); // 'reply to alice@customer.com about ...' +``` + +A token is `tok__`: the entity lowercased with anything that is not a letter or digit collapsed to `_`, then a 12-character id derived with the pseudonymisation key. It is spelt to survive a model: one word, no punctuation a tokenizer would split on, an entity name a model can reason about. Tokens are stable, so the same address yields the same token in every prompt, and they cannot be guessed. + +The original goes into the token store, encrypted with the application key, for `ttl` seconds. The operator's `ttl` option overrides the global `tokenization.ttl`; a `ttl` of null keeps the original forever. + +### Detokenizing + +`Redactor::detokenize()` walks a string or an array and replaces every token the store knows: + +```php +Redactor::detokenize('reply to tok_email_k4m9rp2xzq'); // 'reply to alice@customer.com' +Redactor::detokenize(['text' => 'tok_email_k4m9rp2xzq', 'n' => 1]); +``` + +A token the store does not know, whether expired, from another application, or invented by the model, is left exactly as it is, since guessing would be worse. Content that is neither a string nor an array is returned unchanged. + +### The Token Store + +Originals live in the application cache under the `redactor:token:` prefix, encrypted with `APP_KEY` before they are written: + +```php +'tokenization' => [ + 'store' => env('REDACTOR_TOKEN_STORE'), // null for the default cache store + 'ttl' => env('REDACTOR_TOKEN_TTL', 86_400), +], +``` + +A cache entry that fails to decrypt, after a key rotation or corruption, is treated as unknown. The store is resolved lazily the first time a `tokenize` operator runs, so the redactor can be built before the cache and encrypter are ready. + +`Kirschbaum\Redactor\Tokenization\TokenStore` is the contract for another backing store, a vault or a table. Bind your implementation to that interface in the container. See [Extending](extending.md#token-stores). + +## Trust Model + +Three things hold the mapping between a stand-in and its original, and they carry different trust: + +| Stand-in | Reversible by | Needs | +| --- | --- | --- | +| `hash`, `surrogate` | nobody | the key, to confirm a guess | +| `tokenize` | the application | the token store and `APP_KEY` | +| `redact`, `mask`, `partial`, `remove`, `nullify` | nobody | nothing | + +Anyone holding the cache and the application key can resolve tokens, which is the trust the application itself already carries. Anyone holding the pseudonymisation key can confirm whether a given email produced a given surrogate, which is why the key stays with the application and never with the logs. Tokens and surrogates are derived with the same key, so rotating it invalidates both. diff --git a/docs/rules.md b/docs/rules.md new file mode 100644 index 0000000..06b4763 --- /dev/null +++ b/docs/rules.md @@ -0,0 +1,441 @@ +# Rules + +- [Introduction](#introduction) +- [Pattern Rules](#pattern-rules) + - [Shorthand and Full Form](#shorthand-and-full-form) + - [Modes](#modes) + - [Capture Groups](#capture-groups) + - [Validators](#validators) + - [Keywords](#keywords) + - [Minimum Length](#minimum-length) + - [Samples](#samples) + - [Dictionary Rules](#dictionary-rules) + - [Per-Rule Allow Lists](#per-rule-allow-lists) + - [Entity and Operator](#entity-and-operator) +- [Allow-Lists](#allow-lists) +- [Path Rules](#path-rules) +- [Safe and Blocked Keys](#safe-and-blocked-keys) + - [Wildcards](#wildcards) +- [Known Secrets](#known-secrets) +- [Confidence](#confidence) +- [How Detections Are Resolved](#how-detections-are-resolved) +- [PCRE Failures](#pcre-failures) + +## Introduction + +A rule is anything that tells the redactor a value is sensitive. There are four kinds, and they answer different questions: + +| Kind | Asks | Configured under | +| --- | --- | --- | +| Pattern rule | Does the value look like a secret? | `patterns` | +| Path rule | Is the value at this exact location? | `paths` | +| Key rule | Is the value under a key with this name? | `safe_keys`, `blocked_keys` | +| Known secret | Is one of the application's own credentials in the value? | `known_secrets` | + +A rule only detects. What replaces a detected value is decided by an [operator](operators-and-pseudonymisation.md). + +## Pattern Rules + +### Shorthand and Full Form + +A pattern is a named entry under `patterns`. The shorthand is a bare regex, and the matched span is replaced with the profile's replacement string: + +```php +'patterns' => [ + 'internal_id' => '/\bINT-\d{8}\b/', +], +``` + +The full form is an array with a `pattern` and any of the options below: + +```php +'patterns' => [ + 'credit_card' => [ + 'pattern' => '/\b(?:\d[ -]*?){13,16}\b/', + 'mode' => 'partial', + 'keep' => 4, + 'mask_character' => '*', + 'validator' => 'luhn', + 'entity' => 'credit_card', + 'confidence' => 0.6, + 'keywords' => [], + 'min_length' => 13, + 'samples' => ['4111111111111111'], + 'counter_samples' => ['1234567890123456'], + 'allow' => [], + ], +], +``` + +| Option | Type | Default | Meaning | +| --- | --- | --- | --- | +| `pattern` | `string` | required unless `words` is given | The regex, with delimiters. | +| `words` | `string[]` | | A dictionary instead of a regex. See [Dictionary Rules](#dictionary-rules). | +| `mode` | `string` | `replace` | How to rewrite the match. See [Modes](#modes). | +| `keep` | `int` | `4` | Characters kept by `partial` mode. | +| `mask_character` | `string` | `*` | The character `mask` and `partial` write. Only the first character is used. | +| `capture` | `int` | `0` | The capture group holding the secret. See [Capture Groups](#capture-groups). | +| `validator` | `string` | none | `luhn`, `iban` or `ssn`. See [Validators](#validators). | +| `entity` | `string` | the rule name | What kind of thing the rule finds, for `operators`. | +| `confidence` | `float` | `0.6` | The base score of a bare match. See [Confidence](#confidence). | +| `operator` | `string\|array` | none | An operator the rule chooses for itself. See [Entity and Operator](#entity-and-operator). | +| `keywords` | `string[]` | `[]` | Literals at least one of which must appear in the value. See [Keywords](#keywords). | +| `min_length` | `int` | `1` | The shortest text the pattern can match, in bytes. See [Minimum Length](#minimum-length). | +| `samples` | `string[]` | `[]` | Texts the rule must detect. See [Samples](#samples). | +| `counter_samples` | `string[]` | `[]` | Texts the rule must not detect. | +| `allow` | `string[]` | `[]` | Matches this rule alone should let through. See [Per-Rule Allow Lists](#per-rule-allow-lists). | + +A pattern that does not compile is silently dropped from the profile rather than throwing; `redactor:validate` will then report the rule's samples as undetected, if it has any. A rule with no `pattern` and no `words`, or an unrecognised `mode` or `validator`, is a configuration error and throws. + +### Modes + +`mode` says how the matched span is rewritten: + +| Mode | Result for `4111111111111111` | +| --- | --- | +| `replace` (default) | `[REDACTED]` | +| `mask` | `****************`, length preserved | +| `partial` | `************1111`, the last `keep` characters kept | +| `remove` | deleted | +| `full` | the entire value is replaced, not just the match | + +Character counts are multibyte-aware. A `partial` match no longer than `keep` is masked entirely, since revealing any of it would reveal all of it. + +`mask`, `partial` and `remove` are translated into the operators of the same name. `full` is the pre-1.0 behaviour and condemns the whole value on one match; prefer a blocked key or a path rule where that is what you mean. + +### Capture Groups + +Some patterns need surrounding context to match confidently, but the context is not itself sensitive. Name the group holding the secret and the rest survives: + +```php +'aws_secret_key' => [ + 'pattern' => '/(aws_secret_access_key\s*=\s*)([A-Za-z0-9\/+]{40})/i', + 'capture' => 2, +], + +// aws_secret_access_key = [REDACTED] +``` + +If the named group did not participate in the match, the whole match is used instead. The shipped `url_with_auth` and `bearer_token` rules work this way, so the host of a credential URL and the word `Bearer` stay readable. + +### Validators + +A regex asserts shape only. `/\b(?:\d[ -]*?){13,16}\b/` matches order numbers and concatenated timestamps as readily as cards. A validator asserts that the value could be what the pattern claims, and a match that fails one is left alone: + +| Validator | Check | +| --- | --- | +| `luhn` | The payment card check digit, on 12 to 19 digits. | +| `iban` | ISO 13616 mod-97, on 15 to 34 characters. | +| `ssn` | US allocation rules: area not 000, 666 or 900 and above; group not 00; serial not 0000. | + +```php +Redactor::redact('order 2024010112000001 shipped'); // untouched: fails Luhn +Redactor::redact('paid with 4111111111111111'); // 'paid with ************1111' +``` + +A passing validator also raises the match's confidence. See [Confidence](#confidence). + +### Keywords + +A rule can name literals at least one of which must appear somewhere in the value, compared case-insensitively, before the pattern is tried: + +```php +'email' => ['pattern' => '/[^@\s]+@[^@\s]+/', 'keywords' => ['@']], + +'phone_bare' => [ + 'pattern' => '/(? ['phone', 'tel', 'mobile', 'cell', 'fax'], +], +``` + +Keywords do two jobs. The first is cost: "does this value contain an `@`" is one `str_contains()`, so almost every string in a log payload skips the email regex. The second is precision: a bare ten-digit run is a phone number in a value that says `phone` and a Unix timestamp almost everywhere else. + +The keyword must be in the value, not the key. A rule that should fire because of the key name is a [blocked key](#safe-and-blocked-keys). + +### Minimum Length + +A rule can state the shortest text it could possibly match, in bytes: + +```php +'aws_access_key' => ['pattern' => '/\b(?:AKIA|ASIA)[0-9A-Z]{16}\b/', 'min_length' => 20], +``` + +A shorter value skips the rule with one integer compare. Rules are sorted by `min_length` when the profile is built, so once the subject is shorter than a rule's minimum every rule after it is skipped too. Sorting changes only the order rules are *tried*; declared order still decides an equal-score overlap. + +The number must never exceed the true minimum or the rule misses real matches. When in doubt leave it out. Every shipped rule declares one. + +### Samples + +A rule can carry the texts it exists to catch, and texts it must leave alone: + +```php +'order_ref' => [ + 'pattern' => '/\bORD-\d{6}\b/', + 'samples' => ['ref ORD-123456'], + 'counter_samples' => ['ORD-12', 'ORDER-123456'], +], +``` + +`php artisan redactor:validate` runs every sample through the real detection path, with keywords, minimum length, validator and allow lists applied, and fails when a rule no longer detects a sample or detects a counter-sample. A regex edit that quietly stops matching what it was written for then fails CI instead of an audit. Every shipped rule carries both. + +### Dictionary Rules + +A rule can be a list of words instead of a regex. Product codenames, internal project names, a customer list: things no pattern can express and no model would know. + +```php +'codenames' => ['words' => ['Project Falcon', 'Orion'], 'entity' => 'codename'], +``` + +The words are compiled into one whole-word, case-insensitive alternation, longest first, so `Project Falcon` is one finding rather than two and `Orionids` is left alone. Every other rule option applies. An empty list throws. + +### Per-Rule Allow Lists + +A rule can carry exceptions scoped to that rule alone: + +```php +'email' => ['pattern' => '/[^@\s]+@[^@\s]+/', 'allow' => ['/@example\.com$/']], +``` + +Entries follow the same rules as the profile [allow-list](#allow-lists): a literal compared case-insensitively, or a regex when the entry is delimited like one. A rejected match is simply not a detection, so another rule can still report the same span. + +### Entity and Operator + +`entity` names what kind of thing the rule finds. It defaults to the rule name and is what `operators` keys on, so three phone rules sharing `'entity' => 'phone'` are governed by one `operators.phone` entry. + +A rule can also choose its own operator: + +```php +'card' => [ + 'pattern' => '/\b\d{16}\b/', + 'operator' => ['partial' => ['keep' => 4]], +], +``` + +Only a rule that explicitly sets `operator` or a non-default `mode` outranks the profile's `operators.default`. A rule with no preference defers to it. See [Precedence](operators-and-pseudonymisation.md#precedence). + +## Allow-Lists + +Some values look sensitive and are known not to be: the support address on every page, the sandbox card in every fixture, the example key in the docs. List them rather than weakening the pattern that finds them: + +```php +'allowlist' => [ + 'noreply@example.com', // a literal, trimmed and compared case-insensitively + '/^test-\d+@example\.com$/', // a regex, recognised by its delimiters +], +``` + +The allow-list is checked after detection, whichever detector reported the value: a pattern, entropy, a known secret, a blocked key or a path rule. The rules stay as strong as they were written and an allowed value is simply not a finding. A regex entry that cannot be evaluated allows nothing, since the failure mode of an allow-list is a leak. + +An entry is treated as a regex when its first character is not alphanumeric, a backslash or whitespace, and the matching closing delimiter appears at the end, optionally followed by modifiers. Anything else is a literal. + +## Path Rules + +A path says exactly where a value lives. Every other rule infers that from a key name or from the content. + +```php +'paths' => [ + 'request.headers.authorization' => 'redact', + 'user.*.email' => 'surrogate', + '**.password' => 'redact', + 'users[*].token' => 'redact', + 'debug' => 'preserve', +], +``` + +| Segment | Matches | +| --- | --- | +| `literal` | That key exactly, case-insensitively. | +| `*` | Any single level. | +| `**` | Any depth, including none. | +| `[*]` | A list index. `users[*].x` and `users.*.x` are the same pattern. | +| `[0]` | A specific index. `items[0]` and `items.0` are the same. | + +Each value is an operator, in any of the [three spellings](operators-and-pseudonymisation.md#configuring-operators). + +Paths are checked before anything else and, when one matches, *instead of* everything else: no key matching, no pattern scanning, no walk below the matched node. The more specific pattern always wins, scored by segment (a literal counts 3, `*` counts 2, `**` counts 1), so declaration order never matters, and `preserve` carves an exception out of a broader rule without disabling it. + +A path rule on a scalar gets the full operator range and is reported with a certain confidence and the rule name `path:`; the key is its entity. A path rule on an array or object supports `preserve`, `remove` and `nullify`; any other operator collapses the subtree to the replacement string, since masking or pseudonymising an array has no defensible meaning. The profile allow-list still applies to scalars. + +Paths compile once into a trie walked in lockstep with the payload, so the cost tracks the rules currently in play rather than the number configured. Two hundred path rules cost about the same as one. + +## Safe and Blocked Keys + +`blocked_keys` lists keys whose values are always redacted. `safe_keys` lists keys whose values are always preserved. Both are compared case-insensitively. + +```php +'safe_keys' => ['id', 'user_id', 'created_at', '*_count'], +'blocked_keys' => ['password', '*token*', '*key*', 'user_*_data'], +``` + +Two things about `safe_keys` matter more than they look: + +- A safe key preserves the **entire subtree**. `SafeKeysStrategy` ends the chain and stops the walk, so everything nested under a safe key is emitted untouched. Only list keys whose contents cannot carry sensitive data by construction: identifiers, timestamps, enumerations. A free-text field like `message` is not safe because it usually looks harmless. +- `SafeKeysStrategy` runs first in the shipped profiles, so a key listed in both lists is never redacted. `redactor:validate` reports the conflict. + +A value under a blocked key is reported with a certain confidence, the rule name `blocked_key`, and the lowercased key as its entity. That is what lets `operators.email` apply to `['email' => ...]` and to an address inside a message alike. A blocked key holding an array, boolean or null collapses to the replacement string, since there is no text for an operator to act on; `nullify` keeps the key and writes null. + +### Wildcards + +Both lists accept `*`: + +| Pattern | Matches | +| --- | --- | +| `password` | `password` exactly | +| `*token*` | any key containing `token`: `api_token`, `token_data`, `MyTokenField` | +| `password*` | any key starting with `password`: `password_hash`, `password_confirmation` | +| `*_key` | any key ending with `_key`: `private_key`, `api_key` | +| `user_*_token` | `user_api_token`, `user_auth_token` | +| `*` | every key | + +Lists are compiled once, and each pattern becomes the cheapest test for its shape: a hash lookup for exact names, `str_contains()` for `*word*`, `str_starts_with()` and `str_ends_with()` for one-sided wildcards. Only a pattern with an interior wildcard such as `user_*_token` reaches PCRE, compiled once. The shape of your list barely matters in practice. + +## Known Secrets + +Every other detector infers. This one knows: the application's own credentials are already in config, and a log line containing one of them verbatim is a leak whatever it looks like. + +```php +'known_secrets' => [ + 'values' => [env('LEGACY_SIGNING_KEY')], + 'config' => [ + 'app.key', // shipped default + 'services.stripe.secret', + 'database.connections.mysql.password', + 'services.acme', // an array: every string under it + ], +], +``` + +Matching is exact and case-sensitive. Values under 8 characters and nulls are skipped, so an unset secret in a local environment never fails the profile. A registered value is reported with a certain confidence, the rule name `known_secret` and the entity `known_secret`. + +A credential that only exists at runtime is registered the same way, for every profile: + +```php +Redactor::registerSecret($vault->read('signing-key')); +Redactor::registerSecret($token, entity: 'vault_token'); +``` + +`registerSecret()` returns false if the value was too short to register. + +## Confidence + +Binary matching forces a choice between noise and misses: the only way to quieten a rule is to weaken its regex everywhere. Detections carry a score instead. + +```php +'patterns' => [ + 'card' => ['pattern' => '/\b\d{16}\b/', 'confidence' => 0.3, 'validator' => 'luhn'], +], + +'min_confidence' => 0.5, +``` + +The base score comes from the rule. Two signals raise it: + +| Signal | Delta | When | +| --- | --- | --- | +| `validator` | +0.75 | The rule has a validator and the match passed it. | +| `context` | +0.25 | A credential keyword (`secret`, `token`, `password`, `apikey`, `bearer`, `key`, `card`, `cvv`, `ssn` and others) appears in the 40 bytes before the match, or in the key the value sits under. | + +Deltas apply to the remaining headroom rather than adding flat, so signals stack toward 1.0 without exceeding it: 0.3 with a validator becomes 0.825, and with a keyword too 0.87. The same pattern is therefore filtered out as noise on its own and reported when something corroborates it, without editing the pattern. + +Entropy detections start at 0.5, climb by up to 0.4 as the token's entropy clears its threshold, and gain the same context boost. Values found by a blocked key, a path rule or a known secret are certain (1.0). Recognised entities start at the recogniser's own score. `min_confidence` applies to all of them. + +Every finding explains itself. `inspect()` and `redactor:scan` report the score and the signals behind it: + +```json +{ + "rule": "card", + "confidence": 0.87, + "signals": [ + "base +0.30 (pattern \"card\" matched)", + "validator +0.75 (luhn checksum passed)", + "context +0.25 (a credential keyword appears alongside the match)" + ] +} +``` + +Scores map to labels at 0.9 (high), 0.6 (medium) and 0.3 (low); anything lower is very low. + +## How Detections Are Resolved + +The regex, entropy, known-secret and recognition strategies are *detectors*. They report what they found and where, and change nothing. Once every detector in the chain has seen a value, the context resolves the reports and rewrites the original string in one pass: + +1. Detections below `min_confidence` are dropped. +2. Where two detections overlap, the higher score wins. On an equal score the rule declared first in `patterns` wins, then the report that arrived first. +3. Detections whose value is on the allow-list are dropped. +4. Each surviving span is handed to its operator and written into the output left to right. + +Three things follow: + +- An API key beside an email address is not spared because the email matched first. Both are reported, both are rewritten. +- A surrogate written for one detection is never re-detected by the next detector. It has the same shape and entropy as the value it replaced, and a sequential chain would have redacted it again. +- Every finding's offset is a byte offset into the value you passed, so the scanner reports the right column for the second secret on a line. + +Length is deliberately not a criterion in step 2, since it would let a greedy general rule swallow the precise one beside it. A Luhn-validated card (0.6 + 0.75) outranks the bare digit run that also matched it, and `url_with_auth` declared ahead of `email` takes the password out of `https://user:pass@host` and leaves the host. + +A `preserve` operator reports the finding through `inspect()` without marking the payload redacted, which is what a scan that should only report wants. + +## PCRE Failures + +Every regex is evaluated fail-closed. If PCRE gives up on a pattern, because of the backtrack limit, the JIT stack limit or invalid UTF-8, the value is treated as sensitive rather than clean: the whole value is replaced, the failure is logged with the rule name, and the finding is reported with a certain confidence. + +The exceptions are the places where a failure would otherwise excuse a value. An allow-list entry, an entropy exclusion pattern or a safe-key pattern that cannot be evaluated allows nothing. A blocked-key pattern that cannot be evaluated blocks the key. + +## Region Packs + +National identifiers and VAT numbers are grouped by country under `regions` +in `config/redactor.php` and switched on per profile: + +```php +'profiles' => [ + 'default' => [ + 'regions' => ['gb', 'nl', 'eu'], + ], +], +``` + +| Pack | Rules | Checks | +| --- | --- | --- | +| `gb` | National Insurance number, NHS number, VAT | NHS mod-11, VAT mod-97 | +| `nl` | BSN, VAT | eleven-proof, weighted mod-11 | +| `de` | Steuer-ID, VAT | ISO 7064 mod 11,10 | +| `fr` | NIR, VAT | mod-97 key, SIREN key | +| `it` | codice fiscale, VAT | check character, Luhn | +| `es` | DNI and NIE, VAT | mod-23 letter | +| `be` | national register number, VAT | mod-97, both centuries | +| `se` | personnummer, VAT | date plausibility and Luhn | +| `no` | fødselsnummer | two mod-11 control digits | +| `ca` | SIN | Luhn | +| `au` | TFN | weighted mod-11 | +| `eu` | VAT for the remaining member states | format | + +Identifiers whose shape is too common on its own, a nine-digit BSN or SIN, a +ten-digit NHS number, also require a label such as `bsn`, `sin` or `nhs` +somewhere in the value, so an order number is not mistaken for one. Every +pack rule carries samples and counter-samples that `redactor:validate` proves, +and a rule in the profile's own `patterns` with the same name wins over the +pack's. + +Region rules use the entities `national_id`, `health_id` and `vat_number`, so +one operator covers a whole class: + +```php +'operators' => ['national_id' => 'hash', 'vat_number' => 'preserve'], +``` + +## Custom Validators + +Register a validator of your own and name it from any rule: + +```php +use Kirschbaum\Redactor\Patterns\Validator; + +Validator::extend('policy_number', fn (string $value): bool => PolicyNumber::isValid($value)); +``` + +```php +'policy' => ['pattern' => '/\bPOL-\d{8}\b/', 'validator' => 'policy_number'], +``` + +A rule naming a validator that does not exist is a configuration error, so a +typo fails `redactor:validate` rather than silently disabling the check. + diff --git a/docs/scanning.md b/docs/scanning.md new file mode 100644 index 0000000..aa4ed87 --- /dev/null +++ b/docs/scanning.md @@ -0,0 +1,264 @@ +# Scanning + +- [Introduction](#introduction) +- [Scanning Paths](#scanning-paths) +- [Profiles](#profiles) +- [Output Formats](#output-formats) + - [Table](#table) + - [JSON](#json) + - [SARIF](#sarif) + - [JUnit](#junit) +- [Failing the Build](#failing-the-build) +- [Confidence Filtering](#confidence-filtering) +- [Scanning Changes, Not Files](#scanning-changes-not-files) +- [Looking Through Encodings](#looking-through-encodings) +- [Baselines](#baselines) + - [The Ruleset Fingerprint](#the-ruleset-fingerprint) +- [Suppressing a Finding in Place](#suppressing-a-finding-in-place) +- [Verifying Credentials](#verifying-credentials) +- [The Pre-Commit Hook and Workflow](#the-pre-commit-hook-and-workflow) +- [Exit Codes](#exit-codes) +- [Options Reference](#options-reference) + +## Introduction + +`redactor:scan` runs a profile over files instead of payloads and reports where it found something. It is the same detection engine, so a rule you tune for logs is the rule that scans your repository, and every finding's excerpt is taken from the *redacted* text, so reports can be shared without publishing the secrets they report. + +```bash +php artisan redactor:scan +``` + +## Scanning Paths + +Pass files or directories; the default is the application's base path: + +```bash +php artisan redactor:scan path/to/file.txt +php artisan redactor:scan app/ config/ +``` + +Directories are walked with these exclusions: + +- Files matching `scan.exclude_patterns`, tested against the basename and the path relative to the scanned directory. A pattern ending in `/*` prunes the whole directory. +- Files larger than `scan.max_file_size` (10 MB by default). +- Binary files, when `scan.skip_binary` is true: a NUL byte in the first 8 KB, or content that is neither valid UTF-8 nor mostly printable. +- Files git already ignores, when `scan.respect_gitignore` is true. + +A file you name explicitly is scanned even if a pattern would exclude it. A path that does not exist is warned about and skipped. + +Each file is read as overlapping windows of lines (`scan.window_lines` and `scan.overlap_lines`, 512 and 4 by default), so memory stays flat whatever the file size. Windows overlap so a secret spanning a boundary, a PEM block or a wrapped connection string, is still found; the duplicate the overlap produces is dropped by rule, line and column. + +## Profiles + +The scanner uses `scan.profile`, which is `file_scan` unless you change it, or whatever `--profile` names: + +```bash +php artisan redactor:scan --profile=strict app/ +``` + +`file_scan` has no key-based strategies, since a file has no keys, and adds labelled rules such as `password_assignment` and `aws_secret_key` that only make sense in source and config files. See [Configuration](configuration.md#the-shipped-profiles-compared). + +## Output Formats + +### Table + +The default. Findings, not files, ranked by severity so the certain ones are read first: + +``` + Severity Rule Location Excerpt + HIGH aws_access_key app/config.env:3:19 AWS_ACCESS_KEY_ID=[REDACTED] + MEDIUM email app/seed.php:12:24 'contact' => '[REDACTED]', +``` + +Severity comes from confidence: `HIGH` at 0.9 and above, or for findings with no score such as a known secret; `MEDIUM` at 0.6; `LOW` at 0.3; `VERY LOW` below that. A credential confirmed live by [verification](#verifying-credentials) is `LIVE`, above everything else. Pass `--summary-only` to print the totals without the table. + +### JSON + +```bash +php artisan redactor:scan --output=json +``` + +One object per scanned file, with `path`, `ruleset`, `status` (`clean`, `findings` or `skipped`), `findings_count`, `findings`, `profile` and `error`. Each finding carries `rule`, `entity`, `line`, `column`, `excerpt`, `confidence`, `severity`, `signals`, `verification`, `commit`, `encoding`, `profile` and `fingerprint`. Nothing in it is the secret. + +### SARIF + +```bash +php artisan redactor:scan --output=sarif > redactor.sarif +``` + +SARIF 2.1.0, which GitHub code scanning renders inline on the pull request: + +```yaml +- run: php artisan redactor:scan --output=sarif > redactor.sarif +- uses: github/codeql-action/upload-sarif@v3 + with: + sarif_file: redactor.sarif +``` + +Severity maps onto SARIF levels so a low-confidence hit is a note rather than a merge blocker: `high` is `error`, `medium` is `warning`, anything else `note`. Each result carries the finding's fingerprint under `partialFingerprints`, its entity, confidence and signals under `properties`, and the redacted excerpt as the snippet. The ruleset fingerprint is recorded under the tool driver's properties. + +### JUnit + +```bash +php artisan redactor:scan --output=junit +``` + +JUnit XML for a CI dashboard that already renders test results: one test case per scanned file, one failure per finding, a skipped element for a file that could not be read. The excerpt is already redacted. + +Every format other than `table` suppresses the progress lines, so the output can be piped as-is. + +## Failing the Build + +```bash +php artisan redactor:scan --bail +``` + +Exits 1 when any finding survives the baseline and the confidence floor. Without `--bail` the command exits 0 whatever it finds. + +## Confidence Filtering + +```bash +php artisan redactor:scan --min-confidence=0.8 +``` + +Raises the bar without weakening any pattern. The value must be between 0 and 1 and is applied to the profile before scanning, so a low-scoring detection is never acted on rather than filtered out afterwards. Each finding reports its score, its severity and the signals behind it, so the threshold can be chosen on evidence. See [Confidence](rules.md#confidence). + +## Scanning Changes, Not Files + +A gate on commits cares about what is being added, not what was already there. Three modes scan only the lines a change adds, so a pre-existing finding never blocks a commit and a secret is caught on the line that introduces it: + +```bash +php artisan redactor:scan --staged # what is about to be committed +php artisan redactor:scan --diff=origin/main # what the working tree adds over a ref +php artisan redactor:scan --history # every line every commit ever added +php artisan redactor:scan --history=main..HEAD app/ # a range, and a pathspec +``` + +Findings are reported on their real line numbers in the new file. In history mode each finding names the commit that added it, as `abcd1234:path:line:column`, and a secret that a later commit removed is still found: it is still in the repository. + +With a git mode, the `paths` argument is passed to git as a pathspec rather than walked as directories. `scan.exclude_patterns` still apply. The command must run inside a git repository, and a git failure is reported and exits 1. + +The added lines of each change are scanned as one text, so a secret spanning two adjacent added lines is still found. + +## Looking Through Encodings + +A secret in a repository is often not written plainly. A credential URL in a JSON file reads `https:\/\/user:pass@host`, a key in a Kubernetes secret is base64, a token in a query string is percent-encoded. With `scan.decode` on (the default), the scanner decodes one layer deep and scans what comes out: + +| Encoding | What is decoded | +| --- | --- | +| `json` | Lines with JSON string escapes (`\/`, `\"`, `\uXXXX` and the rest). | +| `url` | Percent-encoded runs. | +| `base64` | Tokens of 20 or more base64 characters that contain upper case, lower case and a digit or symbol, and decode to printable text. | + +A finding inside an encoded span reports the encoding, is located at the span's position, and its excerpt is taken from the decoded, redacted text with the encoding as a prefix: `[base64] password=[REDACTED]`. Set `REDACTOR_SCAN_DECODE=false` to switch it off. + +Redaction of live payloads never decodes. That is a cost on every log line for a case the scanner is the right place to catch. + +## Baselines + +A repository with test fixtures or a documented example key can never go green without a baseline, so record what you have accepted and let CI fail only on new findings: + +```bash +php artisan redactor:scan --update-baseline # writes .redactor-baseline.json and exits 0 +php artisan redactor:scan --bail # now fails only on new secrets +``` + +The baseline path is `scan.baseline`, `.redactor-baseline.json` in the base path by default, or whatever `--baseline` names. `--update-baseline` needs one or the other. + +The file stores, for each accepted finding, a fingerprint plus the rule and path for a human reading the diff. The fingerprint is a hash of the rule, the path and the secret; the secret itself is never written, and because the line number is not part of it a finding stays accepted when the code around it moves. Only the fingerprint is matched. A baseline that cannot be parsed fails the command. + +### The Ruleset Fingerprint + +Every scan reports a short digest of the rules it ran: the patterns and their options, the entropy settings, the confidence floor and the key lists. It appears in JSON output, in SARIF under the tool's properties, and in the baseline it writes. + +Two runs with the same fingerprint are comparable. A baseline generated under a different fingerprint is warned about, since what it accepted may no longer mean the same thing; review it, or run `--update-baseline`. + +## Suppressing a Finding in Place + +A fixture, a documented example, a sandbox credential: mark the line and the scanner skips it, with the reason next to the code rather than in a baseline file: + +```php +$stripe = 'sk_test_4eC39HqLyjWDarjtT1zdp7dc'; // redactor:allow - Stripe's public test key +``` + +The marker is `redactor:allow`, anywhere on the same line. It suppresses every finding on that line. + +## Verifying Credentials + +A scan of a mature repository turns up hundreds of candidates: expired keys, examples in docs, fixtures, rotated credentials. A list that cannot separate the live ones from the dead is a list nobody triages. Verification asks each provider directly whether a detected credential still works. + +It also sends real secrets to third parties, so nothing happens unless all three of these agree: + +1. `scan.verification.enabled` is true, in the config file, reviewable in a diff. +2. The run passes `--verify`, a human decision per run. +3. The provider is listed under `scan.verification.verifiers`, which says who you are willing to tell. + +```php +'verification' => [ + 'enabled' => env('REDACTOR_SCAN_VERIFY', false), + 'verifiers' => ['github_token', 'stripe_key', 'slack_token'], +], +``` + +```bash +php artisan redactor:scan --verify +``` + +An empty `verifiers` list means none: enabling the feature and choosing who to trust with the secrets are separate decisions. Passing `--verify` with verification disabled or with an empty list fails the command with a message saying so. Before contacting anyone, the command names every host it will contact. Redaction itself can never trigger verification; only the scan command can, because nothing running unattended inside an application should be making outbound calls with secrets in them. + +The shipped verifiers: + +| Name | Host | Checks | +| --- | --- | --- | +| `github_token` | `api.github.com` | `GET /user`; 401 means dead. | +| `stripe_key` | `api.stripe.com` | `GET /v1/balance`, read-only; 401 means dead. | +| `slack_token` | `slack.com` | `POST /api/auth.test`; the `ok` field decides, since Slack answers 200 either way. | +| `openai_key` | `api.openai.com` | `GET /v1/models`; 401 means dead. | +| `anthropic_key` | `api.anthropic.com` | `GET /v1/models`; 401 means dead. | +| `sendgrid_key` | `api.sendgrid.com` | `GET /v3/scopes`, which sends nothing; 401 or 403 means dead. | +| `google_api_key` | `generativelanguage.googleapis.com` | `GET /v1/models?key=`; 400 means invalid, 403 means real but not enabled for that API, so still live. | + +Each result is `active`, `inactive` or `unknown`. A confirmed-live credential is ranked `LIVE` (critical) above everything else. A check that could not complete is `unknown` and stays `high`, not `low`: failing to verify is not evidence of safety. A verifier that throws degrades its finding to `unknown` rather than abandoning the scan. The secret never reaches a finding, so it cannot escape through JSON, SARIF or a baseline. + +To add a verifier, implement `Verification\Verifier` and call `SecretVerifier::register($verifier)`; it still has to be allow-listed by name to run. See [Extending](extending.md#verifiers). + +## The Pre-Commit Hook and Workflow + +Publish the hook and the workflow: + +```bash +php artisan vendor:publish --tag=redactor-ci +git config core.hooksPath .githooks +``` + +This writes two files: + +- `.githooks/pre-commit` runs `php artisan redactor:scan --staged --bail` before every commit, so only the secret being committed now can fail it. Accept pre-existing findings into the baseline or mark them `redactor:allow`. +- `.github/workflows/redactor-scan.yml` runs on pull requests and on pushes to `main`. On a pull request it scans the branch's changes over its base (`--diff=origin/ --bail --output=sarif`); on `main` it scans the whole tree against the committed baseline. Either way it uploads the SARIF to GitHub code scanning, so findings render inline on the diff. It checks out with `fetch-depth: 0`, which the diff and history modes need. + +## Exit Codes + +| Code | When | +| --- | --- | +| `0` | The scan completed. With `--bail`, nothing was found beyond the baseline. `--update-baseline` wrote the file. | +| `1` | `--bail` and at least one finding. Also: an unknown `--output`, a `--min-confidence` outside 0 to 1, a baseline that could not be parsed, a profile that could not be resolved, `--verify` without verification enabled and a verifier listed, a git failure or a path outside a git repository in a git mode, `--update-baseline` without a path or that could not be written. | + +## Options Reference + +``` +redactor:scan + {paths?*} Paths to scan (files or directories, defaults to base_path); with a git mode, a pathspec + {--profile=file_scan} Redaction profile to use + {--bail} Exit with code 1 if findings are detected + {--summary-only} Do not display per-file results + {--output=table} Output format (table|json|sarif|junit) + {--staged} Scan only the lines staged for commit + {--diff=} Scan only the lines the working tree adds over this ref, e.g. origin/main + {--history=} Scan the lines added by every commit, optionally in a range like main..HEAD + {--min-confidence=} Ignore findings scoring below this (0-1) + {--verify} Check detected credentials against their providers (sends them off this machine) + {--baseline=} Path to a baseline file of accepted findings + {--update-baseline} Write the current findings to the baseline file and exit 0 +``` + +`--staged`, `--diff` and `--history` are checked in that order; the first one present wins. diff --git a/docs/testing.md b/docs/testing.md new file mode 100644 index 0000000..d320f93 --- /dev/null +++ b/docs/testing.md @@ -0,0 +1,134 @@ +# Testing + +- [Introduction](#introduction) +- [Faking the Redactor](#faking-the-redactor) + - [Assertions](#assertions) + - [Inspecting Calls](#inspecting-calls) +- [Validating Profiles](#validating-profiles) +- [Rule Samples](#rule-samples) +- [Testing Surrogates](#testing-surrogates) +- [The Package's Own Tests](#the-packages-own-tests) + +## Introduction + +Redaction is a runtime promise, and a promise nobody tests is one that quietly stops being kept. The package gives you three ways to test it: a fake that records every call so a test can prove a secret never left, a command that resolves every profile so a bad one fails the deploy, and samples on rules so a regex edit that stops matching fails CI. + +## Faking the Redactor + +`Redactor::fake()` swaps the bound redactor for one that still redacts but remembers every call: + +```php +use Kirschbaum\Redactor\Facades\Redactor; + +$fake = Redactor::fake(); + +$this->postJson('/login', ['email' => 'bob@example.com', 'password' => 'hunter2']); + +$fake->assertNeverEmitted('hunter2', 'bob@example.com'); +$fake->assertRedacted('password'); +$fake->assertFinding('email'); +$fake->assertProfileUsed('strict'); +``` + +The fake redacts for real, with the real configuration, so what it records is what production would have emitted. Install it before the code under test resolves the redactor. The log tap resolves it when the channel is first built, so `Redactor::fake()` belongs before the first `Log::` call in the test, and usually in `setUp()`. + +`Redactor::fake()` returns the `RedactorFake` instance; the facade also forwards to it, so `Redactor::assertRedacted('password')` works. + +### Assertions + +| Assertion | Passes when | +| --- | --- | +| `assertNeverEmitted(string ...$secrets)` | None of the given strings appears in the output of any recorded call, whichever profile or path it took. Fails if no calls were recorded. | +| `assertRedacted(string $key)` | At least one call redacted something under the given key. | +| `assertNotRedacted(string $key)` | No call redacted anything under the given key. | +| `assertFinding(string $rule)` | At least one call produced a finding from the given rule, such as `email`, `blocked_key`, `shannon_entropy` or `known_secret`. | +| `assertSomethingRedacted()` | At least one call changed something. | +| `assertNothingRedacted()` | No call changed anything. | +| `assertProfileUsed(string $profile)` | At least one call ran with the given profile name. A call with no profile counts as `default`. | +| `assertCalled(int $times)` | Exactly that many calls were recorded. | +| `assertNotCalled()` | No calls were recorded. | + +`assertNeverEmitted()` is the strongest thing a test can say about redaction: not "this key was handled" but "this secret did not get out", across every call. Output is compared as a string; arrays are JSON-encoded first. + +### Inspecting Calls + +```php +$fake->recorded(); // every call, oldest first: ['profile' => ?string, 'input' => mixed, 'result' => RedactionResult] +$fake->forget(); // clear the recording +``` + +Each recorded result is the `RedactionResult` the call returned, with its `findings`, so a test can make finer assertions than the built-in ones. + +## Validating Profiles + +```bash +php artisan redactor:validate +``` + +For every configured profile the command: + +1. Resolves the profile, which throws on any invalid value with the config path in the message. +2. Reports any key listed in both `safe_keys` and `blocked_keys`. `SafeKeysStrategy` runs first, so such a key is silently never redacted. +3. Reports any entry in `strategies` that is neither a class implementing `Strategy` nor a registered custom strategy name. +4. Runs every rule's `samples` and `counter_samples` through the real detection path. + +Output is one line per profile with `OK` or the error, and the command exits 1 if any profile fails. Run it in CI and at deploy time; a broken profile otherwise throws at log time, where it is replaced rather than emitted. + +The same check is available in code as `Redactor::validateProfiles()`, which returns `profile => error message` for the broken ones. + +## Rule Samples + +Every rule can carry the texts it must detect and the texts it must leave alone: + +```php +'order_ref' => [ + 'pattern' => '/\bORD-\d{6}\b/', + 'samples' => ['ref ORD-123456'], + 'counter_samples' => ['ORD-12', 'ORDER-123456'], +], +``` + +`redactor:validate` checks each sample with the rule's keywords, minimum length, validator and allow lists applied, and names the rule and the sample that failed: + +``` +rule "order_ref" does not detect its sample "ref ORD-12" +rule "digits" detects its counter-sample "started at 1694600000" +``` + +A sample is checked against the rule that carries it, so another rule matching the same text neither passes nor fails it. Every shipped rule carries both, which is how the package's own test suite proves the shipped patterns still catch what they were written for. + +## Testing Surrogates + +Surrogates are only stable for a given key, so a test that asserts a specific surrogate needs a known one: + +```php +config()->set('redactor.pseudonymization.key', 'a-test-pseudonymization-key-of-sufficient-length'); + +expect(Redactor::redact('alice@customer.com', 'observability')) + ->toBe(Redactor::redact('alice@customer.com', 'observability')); +``` + +Most tests need only stability, which the example above asserts without hard-coding a surrogate. Without a key, and with `APP_KEY` empty in the test environment, the pseudonymising operators fall back to `[REDACTED]`. + +## The Package's Own Tests + +If you contribute to the package, the suite is Pest on Orchestra Testbench: + +```bash +composer test # full suite, in parallel +composer test-coverage # with the 100% coverage floor enforced +composer lint # Pint, Rector, PHPStan (level 10, no baseline) +composer rector:check # what Rector would change, without changing it +composer mutate # mutation testing (Pest); local only, not run in CI +composer preflight # everything CI runs +``` + +Coverage and mutation testing need a coverage driver (pcov or Xdebug) loaded in the CLI; without one Pest reports no coverage and generates no mutations. Both scripts raise the memory limit, which the coverage report needs. + +Conventions worth knowing: + +- Tests live under `tests/Feature`, `tests/Unit` and `tests/Performance`, all bound to `Tests\TestCase`, which registers the service provider and calls `Http::preventingStrayRequests()` so no test can reach a real provider or recogniser. +- `testPseudonymizationKey()` in `tests/Pest.php` is the fixed key every suite uses for surrogate assertions. +- `runningWithCoverage()` lets timing-sensitive tests skip themselves under instrumentation, which flattens the difference between a fast and a slow implementation. +- Every redaction threshold has a boundary test at its edge, since on a redactor an off-by-one is the difference between catching a secret and emitting it. +- The pre-commit hook runs the same checks as `composer preflight`; `composer install` wires it up through `core.hooksPath`. diff --git a/docs/upgrading.md b/docs/upgrading.md new file mode 100644 index 0000000..1387ec5 --- /dev/null +++ b/docs/upgrading.md @@ -0,0 +1,155 @@ +# Upgrading + +- [Introduction](#introduction) +- [Requirements](#requirements) +- [Renamed Classes and Methods](#renamed-classes-and-methods) +- [Removed Methods](#removed-methods) +- [Behaviour Changes](#behaviour-changes) +- [Configuration Keys That Changed Meaning](#configuration-keys-that-changed-meaning) +- [Renamed Rules](#renamed-rules) +- [The Scan Command](#the-scan-command) +- [Checklist](#checklist) + +## Introduction + +This page covers upgrading from 0.1.0 to the next release. Every rename is listed with its replacement, every behaviour change with what to do about it. The old names are gone rather than deprecated, so an upgrade that compiles is an upgrade that has been done. + +If you followed the unreleased `main` branch between 0.1.0 and this release, the interim names it carried, `redactWithMetadata()` among them, are listed too. + +## Requirements + +| | 0.1.0 | Now | +| --- | --- | --- | +| PHP | 8.3, 8.4 | 8.3, 8.4, 8.5 | +| Laravel | 11, 12 | 12, 13 | + +Laravel 11 support is dropped: every 11.x release is flagged by a Packagist security advisory, so Composer's default policy refuses to install any of them. + +`spatie/laravel-package-tools` is no longer required; `symfony/finder`, `symfony/process` and `monolog/monolog` are declared directly. `laravel/mcp` and `laravel/ai` are suggested, not required. + +## Renamed Classes and Methods + +| 0.1.0 | Now | Notes | +| --- | --- | --- | +| `Logging\ReadactFormatter` | `Logging\RedactorFormatter` | Can now wrap an inner formatter: `new RedactorFormatter(new JsonFormatter)`. | +| `Logging\CustomLogTap` | `Logging\RedactorFormatterTap` | Kept for channels that want the formatter. Prefer `Logging\RedactorTap`, which adds a processor and leaves the channel's format alone. | +| `Strategies\RedactionStrategyInterface` | `Strategies\Contracts\Strategy` | Same two methods. | +| `Redactor::getAvailableProfiles()` | `Redactor::profiles()` | | +| `Redactor::profileExists()` | `Redactor::hasProfile()` | | +| `Redactor::getStrategies()` | `Redactor::strategies()` | | +| `Redactor::redactWithMetadata()` (interim) | `Redactor::inspect()` | Returns a `RedactionResult` with `value`, `wasRedacted`, `redactedKeys` and `findings`. | + +Update `config/logging.php`: + +```php +// 0.1.0 +'tap' => [Kirschbaum\Redactor\Logging\CustomLogTap::class], + +// Now +'tap' => [Kirschbaum\Redactor\Logging\RedactorTap::class], +``` + +`RedactorTap` accepts a profile after a colon: `RedactorTap::class.':strict'`. + +## Removed Methods + +| Removed | Replacement | +| --- | --- | +| `Redactor::addStrategy()` | List the strategy in the profile's `strategies`, or `registerCustomStrategy()`. | +| `Redactor::removeStrategy()` | Remove it from the profile's `strategies`. | +| `Redactor::calculateShannonEntropy()` | `(new ShannonEntropyStrategy)->calculateShannonEntropy($string)`. | +| `Redactor::isCommonPattern()` | `(new ShannonEntropyStrategy)->isCommonPattern($string, $config)`. | + +The facade no longer forwards `calculateShannonEntropy()` or `isCommonPattern()`. + +## Behaviour Changes + +Each of these changes what an existing configuration produces. Read them before upgrading. + +**Redaction now replaces the matched span, not the whole value.** `redact('User bob@example.com placed order 123')` returns `'User [REDACTED] placed order 123'` rather than `'[REDACTED]'`. If a rule must condemn the whole value on one match, give it `'mode' => 'full'`; if the intent was "this key is always sensitive", a blocked key or a path rule says so directly. + +**Detections are collected and the value rewritten once.** The regex and entropy strategies no longer rewrite the string as they go. They report, the context resolves overlaps and applies the confidence floor, and the original value is rewritten in one pass. Of two overlapping reports the higher score wins, then the rule listed first. A custom strategy that relied on seeing the string *after* the regex strategy rewrote it now sees the original; implement `DetectingStrategy` and report through `$context->collect()` to join the resolution. + +**Findings for a `preserve` operator are reported without marking the payload redacted.** A scan profile that only reports no longer sets `wasRedacted`. + +**`safe_keys` preserves the entire subtree.** Everything nested under a safe key is emitted untouched. The shipped profiles no longer list `message`, `title`, `url`, `path`, `ip`, `user_agent`, `source` or `target` as safe: all of them are free text or personal data, and with them safe the values were emitted verbatim. `session_id` was listed as both safe and blocked and is now blocked only. If you copied the 0.1.0 lists into your own profile, remove those keys; `redactor:validate` reports any key in both lists. + +**Long strings are truncated and scanned, not replaced.** A value over `max_value_length` keeps its head, which the remaining strategies still inspect, followed by `[REDACTED] (String truncated: 65536 characters, 5000 kept)`. Set `'large_string_behavior' => 'redact'` on a profile to restore the old wholesale replacement. + +**Throwables, dates, enums and closures pass through the walk untouched.** A Throwable used to encode to `{}` and reach the formatter as `[]`, losing the stack trace; a Carbon instance was exploded into its `toArray()` components. Key rules still apply, so `['secret' => $enum]` is still redacted. If you relied on a date being turned into an array, convert it yourself before redacting. + +**Blocked keys go through operators.** The key name is the entity, so `operators.email` applies to a value under an `email` key. Findings from a blocked key now carry the value they matched and a certain score. A blocked key holding an array, boolean or null still collapses to the replacement string. + +**The pseudonymisation salt is shared across profiles.** It used to default to the profile name, so two channels on different profiles produced different surrogates for the same user and could not be joined. Surrogates produced before the upgrade will not match surrogates produced after it. Set a profile's own `pseudonymization.salt` to break correlation on purpose. + +**Monolog integration moved to a processor.** `RedactorTap` pushes `RedactorProcessor`, which redacts message, context and extra without touching the channel's output format. Channels that used `CustomLogTap` had their formatter replaced with the package's own line format; with `RedactorTap` they keep the formatter they were configured with, so a channel whose output was the package's format will change format. Use `RedactorFormatterTap` to keep the old behaviour. + +**Scan findings are structured.** Each finding carries rule, line, column and a redacted excerpt instead of one opaque `full_content_redacted` record per file. Anything parsing the JSON output needs updating. See [The Scan Command](#the-scan-command). + +**Invalid configuration now throws** with the offending path named, instead of silently falling back to a default. A value that used to be ignored, such as a non-numeric `max_depth`, now fails the profile. Run `php artisan redactor:validate` before deploying. + +**Redaction metadata no longer corrupts the payload.** `_redacted` is never added to a list, since a string key would turn a JSON array into an object, and a caller's own `_redacted` key is never overwritten. If you read the marker to know whether anything matched, use `inspect()->wasRedacted` instead. + +**Documented environment variables now take effect.** `REDACTOR_MAX_OBJECT_SIZE` was silently ignored and `REDACTOR_SCAN_MAX_FILE_SIZE` crashed the scan command. If either is set in your environment, it now applies. + +**Scanner exclude patterns now work.** `vendor/*` and `node_modules/*` matched nothing in 0.1.0, so every dependency was scanned. They now match, and binary and gitignored files are skipped too. A scan that used to report findings in `vendor/` will stop. + +**PCRE failures fail closed.** A pattern that errors, on the backtrack limit or bad UTF-8, used to let the value through; it now replaces the value and logs the rule. A rule that was silently never matching may now redact everything it is given, which `redactor:validate` and the rule's samples will surface. + +**Entropy is measured per character, not per byte, and can be judged per alphabet.** Non-ASCII text scores lower than before, and the shipped profiles judge hex against 3.0 and base64 against 4.5 through `charset_thresholds`. Entropy detections now carry a score and go through `operators` and `min_confidence`, which they bypassed before. + +**Checksum validators reject values of the right shape that cannot be real.** The shipped `credit_card`, `ssn` and `iban` rules now validate, so an order number that happens to have 16 digits is left alone. + +**Recursion is depth-bounded and cycle-aware.** A self-referencing `toArray()` used to exhaust memory; it is now replaced at `max_depth` or on the first repeated object. + +**The logging path never throws.** A bad profile no longer takes the channel down, and diagnostics cannot re-enter the logger that raised them. + +**`RedactorFormatter::formatBatch()` formats every record.** It used to return only the first. + +## Configuration Keys That Changed Meaning + +| Key | Before | Now | +| --- | --- | --- | +| `safe_keys` | Preserved the scalar; documented wildcards did not work. | Preserves the whole subtree; wildcards work. | +| `patterns.` | A regex whose match condemned the whole value. | A regex whose match is replaced in place, or a full rule with `mode`, `keep`, `mask_character`, `capture`, `validator`, `entity`, `confidence`, `operator`, `keywords`, `min_length`, `samples`, `counter_samples`, `allow` or `words`. | +| `max_value_length` | Replaced the whole string. | Truncates and scans the head; see `large_string_behavior`. | +| `mark_redacted` | Wrote `_redacted` into any array, lists included, overwriting an existing key. | Associative arrays only, never overwrites, never written into HTTP responses or MCP structured content. | +| `shannon_entropy.threshold` | Judged every token, measured per byte. | Judged only tokens whose alphabet has no `charset_thresholds` entry, measured per character. | +| `scan.exclude_patterns` | Matched basenames only. | Matches basename and relative path; `dir/*` prunes the directory. | +| `non_redactable_object_behavior` | Unchanged in meaning, but Throwables and dates were walked like any other object and lost their content. | Throwables, dates, enums and closures pass through whole; the setting applies only to objects that can be neither `toArray()`'d nor JSON-encoded. | + +New keys, all optional with safe defaults: `large_string_behavior`, `max_depth`, `min_confidence`, `operators`, `paths`, `allowlist`, `known_secrets`, `recognition`, `shannon_entropy.charset_thresholds`, per-profile `pseudonymization`; and at the top level `pseudonymization`, `tokenization`, `events`, `scan.skip_binary`, `scan.respect_gitignore`, `scan.window_lines`, `scan.overlap_lines`, `scan.decode`, `scan.verification` and `scan.baseline`. See [Configuration](configuration.md). + +## Renamed Rules + +The shipped patterns were reorganised into two shared lists spread into every profile. If you reference rule names, in a baseline, a listener on `RedactionPerformed`, or `assertFinding()`, these changed: + +| 0.1.0 rule | Profile | Now | +| --- | --- | --- | +| `phone_simple` | `default`, `file_scan` | `phone_formatted`, `phone_e164` and `phone_bare` in every profile. | +| `phone` | `strict` | Removed. It matched any run of seven digits and spaces. | +| `api_key_stripe` | `file_scan` | `stripe_key`, in every profile. | +| `jwt_token` | `file_scan` | `jwt`, in every profile. | +| `jwt` | `strict` | `jwt`, with the shared pattern. | +| `aws_secret_key` | `file_scan` | Still `file_scan` only, but now requires a label such as `aws_secret_access_key =` before it; it used to match any 40-character alphanumeric run. | +| `url_with_auth` | all | Same name; any scheme, and only the password is replaced. | + +Credential rules that were `file_scan` only in 0.1.0, or absent altogether (`bearer_token`, `private_key_block`, `slack_token`, `anthropic_key`, `openai_key`, `google_api_key`, `sendgrid_key`), now run in `default`, `strict` and `observability` as well. Expect more findings from those profiles. + +## The Scan Command + +`redactor:scan` gained `--output=sarif`, `--output=junit`, `--staged`, `--diff`, `--history`, `--min-confidence`, `--verify`, `--baseline` and `--update-baseline`. The `--output=json` structure changed: each file object now carries `ruleset`, `status`, `findings_count` and a `findings` array of structured findings. See [Scanning](scanning.md#output-formats). + +Baselines did not exist in 0.1.0, so there is nothing to migrate; generate one with `--update-baseline` after upgrading. + +## Checklist + +1. Update `composer.json` to PHP 8.3+ and Laravel 12 or 13, and remove any direct dependency on the package's old transitive requirements. +2. Replace `CustomLogTap` with `RedactorTap` in `config/logging.php`, or with `RedactorFormatterTap` if the channel must keep the package's line format. +3. Replace `ReadactFormatter` with `RedactorFormatter` and `RedactionStrategyInterface` with `Strategies\Contracts\Strategy`. +4. Replace `getAvailableProfiles()`, `profileExists()` and `getStrategies()` with `profiles()`, `hasProfile()` and `strategies()`; replace `redactWithMetadata()` with `inspect()`. +5. Replace `addStrategy()` and `removeStrategy()` with profile configuration; call `calculateShannonEntropy()` and `isCommonPattern()` on `ShannonEntropyStrategy`. +6. Re-publish the config (`vendor:publish --tag=redactor-config --force`) or merge the new keys by hand, and remove free-text keys from any `safe_keys` list you copied. +7. Run `php artisan redactor:validate`. +8. If you export logs elsewhere, note that surrogates change once because the salt no longer includes the profile name. +9. Run `php artisan redactor:scan --update-baseline` if the scanner should start green, and publish the hook and workflow with `vendor:publish --tag=redactor-ci`. diff --git a/phpunit.xml.dist b/phpunit.xml.dist index db92607..528217c 100644 --- a/phpunit.xml.dist +++ b/phpunit.xml.dist @@ -3,6 +3,13 @@ xsi:noNamespaceSchemaLocation="vendor/phpunit/phpunit/phpunit.xsd" bootstrap="vendor/autoload.php" colors="true" + cacheDirectory=".phpunit.cache" + failOnWarning="true" + failOnRisky="true" + failOnDeprecation="true" + failOnNotice="true" + failOnEmptyTestSuite="true" + beStrictAboutOutputDuringTests="true" > @@ -11,6 +18,9 @@ tests/Feature + + tests/Performance + diff --git a/rector.php b/rector.php new file mode 100644 index 0000000..c06bca6 --- /dev/null +++ b/rector.php @@ -0,0 +1,24 @@ +withPaths([ + __DIR__.'/src', + __DIR__.'/config', + __DIR__.'/tests', + ]) + ->withPhpSets(php83: true) + ->withPreparedSets( + deadCode: true, + codeQuality: true, + typeDeclarations: true, + earlyReturn: true, + ) + ->withSets([ + LaravelSetList::LARAVEL_CODE_QUALITY, + LaravelSetList::LARAVEL_COLLECTION, + ]); diff --git a/scripts/preflight.sh b/scripts/preflight.sh index 9cd978f..350f4ea 100755 --- a/scripts/preflight.sh +++ b/scripts/preflight.sh @@ -21,6 +21,19 @@ else echo "${GREEN}✨ ✅ Pint passed.${RESET}" fi +# -------------------------------------- +# Step 1b: Rector (Refactoring rules, dry run) +# -------------------------------------- +echo "${YELLOW}🛠 Running Rector...${RESET}" +./vendor/bin/rector process --dry-run --no-progress-bar +if [[ $? -ne 0 ]]; then + echo "${RED}🛠 ⭕ Rector would change files.${RESET}" + echo " ${YELLOW}Run 'composer rector' to apply them, then commit again.${RESET}" + status=1 +else + echo "${GREEN}🛠 ✅ Rector passed.${RESET}" +fi + # -------------------------------------- # Step 2: PHPStan (Static analysis) # -------------------------------------- diff --git a/src/Ai/RedactPrompt.php b/src/Ai/RedactPrompt.php new file mode 100644 index 0000000..3e9a927 --- /dev/null +++ b/src/Ai/RedactPrompt.php @@ -0,0 +1,66 @@ +make(Redactor::class), $profile, $detokenizeResponse); + } + + /** + * @param Closure(AgentPrompt): AgentResponse $next + */ + public function handle(AgentPrompt $prompt, Closure $next): AgentResponse + { + $redacted = $this->redactor->redactSafely($prompt->prompt, $this->profile); + + $revised = $prompt->revise(is_string($redacted) ? $redacted : (string) json_encode($redacted)); + + $response = $next($revised); + + if (! $this->detokenizeResponse) { + return $response; + } + + return $response->then(function (AgentResponse $response): void { + $resolved = $this->redactor->detokenize($response->text); + + if (is_string($resolved)) { + $response->text = $resolved; + } + }); + } +} diff --git a/src/Config/ConfigValue.php b/src/Config/ConfigValue.php new file mode 100644 index 0000000..5bace97 --- /dev/null +++ b/src/Config/ConfigValue.php @@ -0,0 +1,269 @@ + $allowed + */ + public static function enum(mixed $value, array $allowed, string $default, string $path): string + { + $string = self::string($value, $default, $path); + + if (! in_array($string, $allowed, true)) { + throw new ConfigurationException(sprintf( + 'Redactor config [%s] must be one of [%s], got "%s".', + $path, + implode(', ', $allowed), + $string + )); + } + + return $string; + } + + /** + * Coerce the value to a list of strings, dropping anything else. + * + * @return array + */ + public static function stringList(mixed $value, string $path): array + { + if ($value === null) { + return []; + } + + if (! is_array($value)) { + throw new ConfigurationException(sprintf( + 'Redactor config [%s] must be an array, got %s.', + $path, + self::describe($value) + )); + } + + $out = []; + + foreach ($value as $item) { + if (is_string($item)) { + $out[] = $item; + } + } + + return $out; + } + + /** + * Coerce the value to a string-keyed map. + * + * @return array + */ + public static function map(mixed $value, string $path): array + { + if ($value === null) { + return []; + } + + if (! is_array($value)) { + throw new ConfigurationException(sprintf( + 'Redactor config [%s] must be an array, got %s.', + $path, + self::describe($value) + )); + } + + /** @var array $normalised */ + $normalised = []; + + foreach ($value as $key => $item) { + $normalised[(string) $key] = $item; + } + + return $normalised; + } + + /** + * Coerce the value to an integer. + */ + private static function toInt(mixed $value, string $path): int + { + if (is_int($value)) { + return $value; + } + + if (is_float($value) && floor($value) === $value) { + return (int) $value; + } + + if (is_string($value)) { + $trimmed = trim($value); + + // Reject "12abc" and "1.5", which (int) would silently accept... + if (preg_match('/^-?\d+$/', $trimmed) === 1) { + return (int) $trimmed; + } + } + + throw new ConfigurationException(sprintf( + 'Redactor config [%s] must be an integer, got %s.', + $path, + self::describe($value) + )); + } + + /** + * Describe the value for an error message. + */ + private static function describe(mixed $value): string + { + if (is_object($value)) { + return $value::class; + } + + if (is_string($value)) { + return sprintf('string("%s")', $value); + } + + if (is_bool($value)) { + return $value ? 'true' : 'false'; + } + + if (is_scalar($value)) { + return sprintf('%s(%s)', gettype($value), (string) $value); + } + + return gettype($value); + } +} diff --git a/src/Config/ProfileCache.php b/src/Config/ProfileCache.php new file mode 100644 index 0000000..953de7f --- /dev/null +++ b/src/Config/ProfileCache.php @@ -0,0 +1,63 @@ +, shared: array, built: RedactorConfig}> */ + private static array $entries = []; + + private static int $builds = 0; + + /** + * Get a build number no previously built profile has had. + * + * RedactorConfig is readonly and cannot hold the counter itself. + */ + public static function nextBuildId(): int + { + return ++self::$builds; + } + + /** + * Get the cached profile if it was built from the same raw config. + * + * @param array $raw the profile's own config + * @param array $shared package-level settings the profile was built with + */ + public static function get(string $profile, array $raw, array $shared = []): ?RedactorConfig + { + $entry = self::$entries[$profile] ?? null; + + return $entry !== null && $entry['raw'] === $raw && $entry['shared'] === $shared + ? $entry['built'] + : null; + } + + /** + * Cache the built profile against the raw config it was built from. + * + * @param array $raw + * @param array $shared + */ + public static function put(string $profile, array $raw, RedactorConfig $built, array $shared = []): RedactorConfig + { + self::$entries[$profile] = ['raw' => $raw, 'shared' => $shared, 'built' => $built]; + + return $built; + } +} diff --git a/src/Console/Commands/RedactorScanCommand.php b/src/Console/Commands/RedactorScanCommand.php index 5113d73..119532c 100644 --- a/src/Console/Commands/RedactorScanCommand.php +++ b/src/Console/Commands/RedactorScanCommand.php @@ -1,171 +1,445 @@ $paths */ $paths = $this->argument('paths'); - if (empty($paths)) { + if ($paths === []) { $paths = [base_path()]; } /** @var string $profile */ $profile = $this->option('profile') ?? config('redactor.scan.profile', 'default'); - /** @var bool $bail */ - $bail = $this->option('bail'); - - /** @var bool $summaryOnly */ - $summaryOnly = $this->option('summary-only'); + $bail = (bool) $this->option('bail'); + $summaryOnly = (bool) $this->option('summary-only'); /** @var string $outputFormat */ $outputFormat = $this->option('output') ?? 'table'; - $this->components->info('Scanning paths: '.implode(', ', $paths)." with profile: {$profile}"); + if (! in_array($outputFormat, ['table', 'json', 'sarif', 'junit'], true)) { + $this->components->error("Unknown --output format [{$outputFormat}]. Use table, json, sarif or junit."); + + return Command::FAILURE; + } + + $gitMode = $this->gitMode(); + + $minConfidence = $this->option('min-confidence'); + + if (is_string($minConfidence) && $minConfidence !== '') { + if (! is_numeric($minConfidence) || (float) $minConfidence < 0 || (float) $minConfidence > 1) { + $this->components->error('--min-confidence must be a number between 0 and 1.'); + + return Command::FAILURE; + } + + // Applied to the profile rather than filtered afterwards, so a low-scoring detection is never acted on... + config(["redactor.profiles.{$profile}.min_confidence" => (float) $minConfidence]); + } + + $baselinePath = $this->baselinePath(); + $updateBaseline = (bool) $this->option('update-baseline'); + + try { + $baseline = $baselinePath !== null ? Baseline::load($baselinePath) : Baseline::empty(); + } catch (\JsonException $e) { + $this->components->error($e->getMessage()); + + return Command::FAILURE; + } + + // Machine-readable output must not be polluted with progress chatter... + $quiet = $outputFormat !== 'table'; + + try { + $ruleset = RedactorConfig::fromConfig($profile)->rulesetFingerprint; + } catch (ConfigurationException $e) { + $this->components->error($e->getMessage()); + + return Command::FAILURE; + } + + if (! $quiet && $baseline->ruleset !== null && $baseline->ruleset !== $ruleset) { + $this->components->warn(sprintf( + 'The baseline was generated under ruleset %s; this scan runs ruleset %s. Findings it accepted may no longer mean the same thing - review it, or run --update-baseline.', + $baseline->ruleset, + $ruleset + )); + } + + if (! $quiet) { + $this->components->info($gitMode === null + ? 'Scanning paths: '.implode(', ', $paths)." with profile: {$profile}" + : "Scanning {$gitMode} with profile: {$profile}"); + } + + $ignorePatterns = ConfigValue::stringList( + config('redactor.scan.exclude_patterns', []), + 'scan.exclude_patterns' + ); + + // Config::array() and Config::integer() throw when the value arrives as a + // string, which is exactly what env() produces for REDACTOR_SCAN_*... + $maxFileSize = ConfigValue::positiveInt( + config('redactor.scan.max_file_size'), + 10_485_760, + 'scan.max_file_size' + ); + + $skipBinary = ConfigValue::bool(config('redactor.scan.skip_binary'), true, 'scan.skip_binary'); + $respectGitignore = ConfigValue::bool(config('redactor.scan.respect_gitignore'), true, 'scan.respect_gitignore'); + + $scanner = Container::getInstance()->make(Scanner::class); - /** @var array $ignorePatterns */ - $ignorePatterns = Config::array('redactor.scan.exclude_patterns', []); + if ((bool) $this->option('verify')) { + $verifier = SecretVerifier::fromConfig( + ConfigValue::map(config('redactor.scan.verification', []), 'scan.verification') + ); - /** @var int $maxFileSize */ - $maxFileSize = Config::integer('redactor.scan.max_file_size', 10_485_760); + if (! $verifier instanceof SecretVerifier) { + $this->components->error( + 'Verification is not enabled. Set redactor.scan.verification.enabled to true ' + .'and list the providers you permit under redactor.scan.verification.verifiers.' + ); - $files = $this->collectFiles($paths, $ignorePatterns, $maxFileSize); + return Command::FAILURE; + } + + // Verification sends real credentials to third parties, so say so before it happens... + if (! $quiet) { + $this->components->warn(sprintf( + 'Verification is on: detected credentials will be sent to %s.', + implode(', ', $verifier->hosts()) + )); + } - $scanner = resolve(Scanner::class); + $scanner = $scanner->withVerifier($verifier); + } /** @var Collection $results */ $results = collect(); - foreach ($files as $file) { - $result = $scanner->scanFile($file, $profile); - $results->push($result); + if ($gitMode !== null) { + try { + $patches = $this->collectPatches($gitMode, $this->argument('paths'), $ignorePatterns); + } catch (GitException $e) { + $this->components->error($e->getMessage()); + + return Command::FAILURE; + } + + foreach ($patches as $patch) { + $results->push($scanner->scanPatch($patch, $profile)); + } + } else { + $relativeTo = base_path(); + + foreach ($this->collectFiles($paths, $ignorePatterns, $maxFileSize, $skipBinary, $respectGitignore, $quiet) as $file) { + $results->push($scanner->scanFile($file, $profile, $relativeTo)); + } + } + + /** @var Collection $allFindings */ + $allFindings = $results->flatMap(fn (ScanResult $r): array => $r->findings); + + if ($updateBaseline) { + return $this->writeBaseline($baselinePath, $allFindings->all(), $ruleset); + } + + $suppressed = 0; + + if (! $baseline->isEmpty()) { + $before = $allFindings->count(); + $results = $results->map(fn (ScanResult $r): ScanResult => $r->withoutBaseline($baseline->fingerprints)); + $allFindings = $results->flatMap(fn (ScanResult $r): array => $r->findings); + $suppressed = $before - $allFindings->count(); + } + + $this->displayResults($results, $allFindings->all(), $outputFormat, $summaryOnly, $ruleset); + + $filesWithFindings = $results->filter(fn (ScanResult $r): bool => $r->hasFindings()); + + if (! $quiet) { + $this->newLine(); + $this->components->info($gitMode === null + ? "Scan complete. Files scanned: {$results->count()}" + : "Scan complete. Changes scanned: {$results->count()}"); + $this->components->info("Files with findings: {$filesWithFindings->count()}"); + $this->components->info("Total findings: {$allFindings->count()}"); + + if ($suppressed > 0) { + $this->components->info("Suppressed by baseline: {$suppressed}"); + } + } + + return ($bail && $allFindings->isNotEmpty()) ? Command::FAILURE : Command::SUCCESS; + } + + /** + * Get the requested git mode, described for the operator. + */ + protected function gitMode(): ?string + { + if ((bool) $this->option('staged')) { + return 'staged changes'; + } + + $diff = $this->option('diff'); + + if (is_string($diff) && $diff !== '') { + return "changes over {$diff}"; + } + + if ($this->input->hasParameterOption('--history')) { + $range = $this->option('history'); + + return is_string($range) && $range !== '' ? "history {$range}" : 'full history'; + } + + return null; + } + + /** + * Collect the patches the chosen git mode produces, minus excluded paths. + * + * @param array $pathspec + * @param array $ignorePatterns + * @return array + * + * @throws GitException when this is not a git repository or git fails + */ + protected function collectPatches(string $mode, array $pathspec, array $ignorePatterns): array + { + $git = new GitRepository(base_path()); + + if (! $git->isRepository()) { + throw new GitException('['.base_path().'] is not inside a git repository.'); + } + + $patches = match (true) { + (bool) $this->option('staged') => $git->staged($pathspec), + is_string($this->option('diff')) && $this->option('diff') !== '' => $git->diff($this->option('diff'), $pathspec), + default => $git->history(is_string($this->option('history')) ? $this->option('history') : null, $pathspec), + }; + + return array_values(array_filter( + $patches, + fn (Patch $patch): bool => ! FileCollector::matchesExclude($patch->path, $ignorePatterns) + )); + } + + protected function baselinePath(): ?string + { + /** @var string|null $option */ + $option = $this->option('baseline'); + + if (is_string($option) && $option !== '') { + return $option; + } + + $configured = config('redactor.scan.baseline'); + + return is_string($configured) && $configured !== '' ? $configured : null; + } + + /** + * @param array $findings + */ + protected function writeBaseline(?string $path, array $findings, ?string $ruleset = null): int + { + if ($path === null) { + $this->components->error('--update-baseline needs a path: pass --baseline= or set redactor.scan.baseline.'); + + return Command::FAILURE; } - $this->displayResults($results, $outputFormat, $summaryOnly); + if (! Baseline::write($path, $findings, now()->toIso8601String(), $ruleset)) { + $this->components->error("Could not write baseline file [{$path}]."); - $findings = $results->filter(fn (ScanResult $r) => $r->hasFindings()); + return Command::FAILURE; + } - $this->newLine(); - $this->components->info("Scan complete. Files scanned: {$results->count()}"); - $this->components->info("Files with findings: {$findings->count()}"); + $this->components->info(sprintf('Wrote %d accepted findings to %s', count($findings), $path)); - return ($bail && $findings->count() > 0) ? Command::FAILURE : Command::SUCCESS; + return Command::SUCCESS; } /** - * Collect files from the given paths (files or directories). + * Collect the files to scan from the given paths. * * @param array $paths * @param array $ignorePatterns * @return array */ - protected function collectFiles(array $paths, array $ignorePatterns, int $maxFileSize): array - { - // Check for non-existent paths and warn user + protected function collectFiles( + array $paths, + array $ignorePatterns, + int $maxFileSize, + bool $skipBinary = true, + bool $respectGitignore = true, + bool $quiet = false + ): array { + // Warn about paths that do not exist... $validPaths = []; foreach ($paths as $path) { if (is_file($path) || is_dir($path)) { $validPaths[] = $path; - } else { + } elseif (! $quiet) { $this->components->warn("Path not found or not accessible: {$path}"); } } - // Let FileCollector handle all the filtering logic return FileCollector::collect( paths: $validPaths, excludePatterns: $ignorePatterns, - maxSizeBytes: $maxFileSize + maxSizeBytes: $maxFileSize, + skipBinary: $skipBinary, + respectGitignore: $respectGitignore ); } /** - * Display scan results in the specified format. + * Display the scan results in the given format. * * @param Collection $results + * @param array $findings */ - protected function displayResults(Collection $results, string $format, bool $summaryOnly): void + protected function displayResults(Collection $results, array $findings, string $format, bool $summaryOnly, ?string $ruleset = null): void { - if ($format === 'json') { - $this->displayJsonResults($results); - } else { - $this->displayTableResults($results, $summaryOnly); - } + match ($format) { + 'json' => $this->displayJsonResults($results, $ruleset), + 'sarif' => $this->displaySarifResults($findings, $ruleset), + 'junit' => $this->output->writeln(JunitReport::build($results->all())), + default => $this->displayTableResults($results, $findings, $summaryOnly), + }; } /** - * Display results in JSON format. - * * @param Collection $results */ - protected function displayJsonResults(Collection $results): void + protected function displayJsonResults(Collection $results, ?string $ruleset = null): void { - $jsonData = $results->map(fn (ScanResult $r) => [ + $jsonData = $results->map(fn (ScanResult $r): array => [ 'path' => $r->path, + 'ruleset' => $ruleset, 'status' => $r->skipped ? 'skipped' : ($r->hasFindings() ? 'findings' : 'clean'), 'findings_count' => count($r->findings), - 'findings' => $r->findings, + 'findings' => array_map(fn (ScanFinding $f): array => $f->toArray(), $r->findings), 'profile' => $r->profile, 'error' => $r->error, - ])->toArray(); + ])->all(); + + $jsonOutput = json_encode($jsonData, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES); - $jsonOutput = json_encode($jsonData, JSON_PRETTY_PRINT); if ($jsonOutput !== false) { $this->output->writeln($jsonOutput); } } /** - * Display results in table format. - * + * @param array $findings + */ + protected function displaySarifResults(array $findings, ?string $ruleset = null): void + { + $sarif = json_encode(SarifReport::build($findings, '1.0.0', $ruleset), JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES); + + if ($sarif !== false) { + $this->output->writeln($sarif); + } + } + + /** * @param Collection $results + * @param array $findings */ - protected function displayTableResults(Collection $results, bool $summaryOnly): void + protected function displayTableResults(Collection $results, array $findings, bool $summaryOnly): void { if ($summaryOnly) { return; } - $tableData = $results->map(function (ScanResult $result) { - $status = $result->skipped - ? 'SKIPPED' - : ($result->hasFindings() ? 'FINDINGS' : 'CLEAN'); + if ($findings === []) { + $this->components->info(sprintf('No findings across %d files.', $results->count())); - $findingsCount = $result->skipped ? '-' : (string) count($result->findings); + return; + } - $path = $result->path; - // Truncate very long paths for better table display - if (strlen($path) > 60) { - $path = '...'.substr($path, -57); - } + // Findings, not files: a list of file names with a count next to each is nothing + // you can act on. Sorted by severity so the certain findings are read first... + $rank = fn (ScanFinding $f): int => match ($f->severity()) { + 'critical' => 4, 'high' => 3, 'medium' => 2, 'low' => 1, default => 0, + }; - return [ - 'Status' => $status, - 'Findings' => $findingsCount, - 'File Path' => $path, - ]; - })->toArray(); + usort($findings, fn (ScanFinding $a, ScanFinding $b): int => [$rank($b), $b->confidence ?? 1.0] + <=> [$rank($a), $a->confidence ?? 1.0]); - $this->table(['Status', 'Findings', 'File Path'], $tableData); + $this->table( + ['Severity', 'Rule', 'Location', 'Excerpt'], + array_map(fn (ScanFinding $f): array => [ + match ($f->severity()) { + 'critical' => 'LIVE', + 'high' => 'HIGH', + 'medium' => 'MEDIUM', + 'low' => 'LOW', + default => 'VERY LOW', + }, + $f->rule, + $this->shorten($f->location(), 52), + $this->shorten($f->excerpt, 48), + ], $findings) + ); + + $skipped = $results->filter(fn (ScanResult $r): bool => $r->skipped); + + foreach ($skipped as $result) { + $this->components->warn("Skipped {$result->path}: {$result->error}"); + } + } + + private function shorten(string $value, int $limit = 60): string + { + return strlen($value) > $limit ? '...'.substr($value, -($limit - 3)) : $value; } } diff --git a/src/Console/Commands/RedactorValidateCommand.php b/src/Console/Commands/RedactorValidateCommand.php new file mode 100644 index 0000000..e98bb9b --- /dev/null +++ b/src/Console/Commands/RedactorValidateCommand.php @@ -0,0 +1,58 @@ +profiles(); + + if ($profiles === []) { + $this->components->error('No redaction profiles are configured.'); + + return Command::FAILURE; + } + + $errors = $redactor->validateProfiles(); + + foreach ($profiles as $profile) { + if (isset($errors[$profile])) { + $this->components->twoColumnDetail( + "{$profile}", + "{$errors[$profile]}" + ); + } else { + $this->components->twoColumnDetail($profile, 'OK'); + } + } + + $this->newLine(); + + if ($errors !== []) { + $this->components->error(sprintf( + '%d of %d profiles are invalid. Fix them before deploying; a broken profile throws at log time.', + count($errors), + count($profiles) + )); + + return Command::FAILURE; + } + + $this->components->info(sprintf('All %d redaction profiles resolve cleanly.', count($profiles))); + + return Command::SUCCESS; + } +} diff --git a/src/Detection/Confidence.php b/src/Detection/Confidence.php new file mode 100644 index 0000000..dd05c68 --- /dev/null +++ b/src/Detection/Confidence.php @@ -0,0 +1,100 @@ + $signals + */ + private function __construct( + public float $score, + public array $signals = [], + ) {} + + /** + * Create a new confidence instance from a base score. + */ + public static function of(float $score, string $reason = 'base rule confidence'): self + { + $clamped = self::clamp($score); + + return new self($clamped, [new Signal('base', $clamped, $reason)]); + } + + /** + * Add a contribution and re-derive the score. + * + * Deltas are applied to the remaining headroom rather than added flat, so + * signals stack toward certainty without ever exceeding it and without one + * strong signal swamping the rest. + */ + public function with(string $name, float $delta, string $reason): self + { + $score = $delta >= 0.0 + ? $this->score + (1.0 - $this->score) * $delta + : $this->score * (1.0 + $delta); + + return new self( + self::clamp($score), + [...$this->signals, new Signal($name, $delta, $reason)], + ); + } + + /** + * Determine if the score meets the given threshold. + */ + public function meets(float $threshold): bool + { + return $this->score >= $threshold; + } + + /** + * Get a description of each signal behind the score. + * + * @return array + */ + public function explain(): array + { + return array_map(fn (Signal $s): string => $s->describe(), $this->signals); + } + + /** + * Get the human-readable label for the score. + */ + public function label(): string + { + return match (true) { + $this->score >= 0.9 => 'high', + $this->score >= 0.6 => 'medium', + $this->score >= 0.3 => 'low', + default => 'very-low', + }; + } + + /** + * Clamp the score to the unit interval. + */ + private static function clamp(float $score): float + { + return max(0.0, min(1.0, $score)); + } +} diff --git a/src/Detection/Detection.php b/src/Detection/Detection.php new file mode 100644 index 0000000..553500c --- /dev/null +++ b/src/Detection/Detection.php @@ -0,0 +1,80 @@ +value); + } + + /** + * Get the offset just past the sensitive span. + */ + public function end(): int + { + return $this->offset + $this->length(); + } + + /** + * Create a detection covering the whole subject because the detector failed. + */ + public static function failClosed(string $entity, string $rule, string $subject, string $key, string $reason): self + { + return new self( + entity: $entity, + rule: $rule, + offset: 0, + value: $subject, + confidence: Confidence::of(Confidence::CERTAIN, $reason), + key: $key, + failClosed: true, + ); + } + + /** + * Determine if this detection covers the same ground as another. + */ + public function overlaps(self $other): bool + { + return $this->offset < $other->end() && $other->offset < $this->end(); + } +} diff --git a/src/Detection/DetectionSet.php b/src/Detection/DetectionSet.php new file mode 100644 index 0000000..f8089da --- /dev/null +++ b/src/Detection/DetectionSet.php @@ -0,0 +1,69 @@ + $detections + * @return array non-overlapping, ordered by offset + */ + public static function resolve(array $detections, float $minConfidence = 0.0): array + { + $candidates = array_values(array_filter( + $detections, + fn (Detection $d): bool => $d->value !== '' && ($d->failClosed || $d->confidence->meets($minConfidence)) + )); + + if (count($candidates) < 2) { + return $candidates; + } + + $kept = []; + + foreach ($candidates as $i => $candidate) { + $beaten = false; + + foreach ($candidates as $j => $other) { + if ($i === $j || ! $candidate->overlaps($other)) { + continue; + } + + $otherWins = $other->confidence->score > $candidate->confidence->score + || ($other->confidence->score === $candidate->confidence->score + && ($other->priority < $candidate->priority + || ($other->priority === $candidate->priority && $j < $i))); + + if ($otherWins) { + $beaten = true; + break; + } + } + + if (! $beaten) { + $kept[] = $candidate; + } + } + + usort($kept, fn (Detection $a, Detection $b): int => $a->offset <=> $b->offset); + + return $kept; + } +} diff --git a/src/Detection/Detector.php b/src/Detection/Detector.php new file mode 100644 index 0000000..b86270d --- /dev/null +++ b/src/Detection/Detector.php @@ -0,0 +1,26 @@ + offsets relative to $subject as given + */ + public function detect(string $subject, string $key, RedactionContext $context): array; +} diff --git a/src/Detection/EntityFilter.php b/src/Detection/EntityFilter.php new file mode 100644 index 0000000..9edc2ec --- /dev/null +++ b/src/Detection/EntityFilter.php @@ -0,0 +1,91 @@ +|null */ + protected ?array $only = null; + + /** @var array */ + protected array $except = []; + + /** + * Create a filter that allows every entity. + */ + public static function all(): self + { + return new self; + } + + /** + * Allow only the given entities. + * + * @param array $entities + */ + public function only(array $entities): static + { + $this->only = $this->index($entities); + + return $this; + } + + /** + * Allow every entity but the given ones. + * + * @param array $entities + */ + public function except(array $entities): static + { + $this->except = [...$this->except, ...$this->index($entities)]; + + return $this; + } + + /** + * Determine if the filter allows every entity. + */ + public function allowsEverything(): bool + { + return $this->only === null && $this->except === []; + } + + /** + * Determine if a detection of this entity may be acted on. + */ + public function allows(string $entity): bool + { + $entity = strtolower($entity); + + if (isset($this->except[$entity])) { + return false; + } + + return $this->only === null || isset($this->only[$entity]); + } + + /** + * @param array $entities + * @return array + */ + private function index(array $entities): array + { + $index = []; + + foreach ($entities as $entity) { + $index[strtolower(trim($entity))] = true; + } + + return $index; + } +} diff --git a/src/Detection/KeywordContext.php b/src/Detection/KeywordContext.php new file mode 100644 index 0000000..33ca732 --- /dev/null +++ b/src/Detection/KeywordContext.php @@ -0,0 +1,87 @@ + */ + public const KEYWORDS = [ + 'secret', 'token', 'password', 'passwd', 'apikey', 'api_key', 'api-key', + 'credential', 'private', 'auth', 'bearer', 'key', 'card', 'cvv', 'ssn', + ]; + + /** + * Add the context signal to a score when the surroundings corroborate it. + */ + public static function boost(Confidence $confidence, string $subject, int $offset, string $key): Confidence + { + if (self::nearby($subject, $offset) || self::keyLooksSensitive($key)) { + return $confidence->with('context', self::BOOST, 'a credential keyword appears alongside the match'); + } + + return $confidence; + } + + /** + * Determine if a credential keyword sits just before the match. + */ + public static function nearby(string $subject, int $offset): bool + { + $start = max(0, $offset - self::WINDOW); + $window = strtolower(substr($subject, $start, $offset - $start)); + + if ($window === '') { + return false; + } + + foreach (self::KEYWORDS as $keyword) { + if (str_contains($window, $keyword)) { + return true; + } + } + + return false; + } + + /** + * Determine if the key itself contains a credential keyword. + */ + public static function keyLooksSensitive(string $key): bool + { + if ($key === '') { + return false; + } + + $lower = strtolower($key); + + foreach (self::KEYWORDS as $keyword) { + if (str_contains($lower, $keyword)) { + return true; + } + } + + return false; + } +} diff --git a/src/Detection/Signal.php b/src/Detection/Signal.php new file mode 100644 index 0000000..6742ef0 --- /dev/null +++ b/src/Detection/Signal.php @@ -0,0 +1,29 @@ +name, $this->delta, $this->reason); + } +} diff --git a/src/Events/RedactionPerformed.php b/src/Events/RedactionPerformed.php new file mode 100644 index 0000000..3b9a9da --- /dev/null +++ b/src/Events/RedactionPerformed.php @@ -0,0 +1,27 @@ + $redactedKeys keys that were redacted, deduplicated + * @param array $rules rule name => number of findings + * @param array $entities entity => number of findings + */ + public function __construct( + public string $profile, + public array $redactedKeys, + public array $rules, + public array $entities, + public int $findings, + ) {} +} diff --git a/src/Exceptions/ConfigurationException.php b/src/Exceptions/ConfigurationException.php new file mode 100644 index 0000000..f1c3108 --- /dev/null +++ b/src/Exceptions/ConfigurationException.php @@ -0,0 +1,12 @@ + getAvailableProfiles() - * @method static bool profileExists(string $profile) - * @method static array<\Kirschbaum\Redactor\Strategies\RedactionStrategyInterface> getStrategies(?string $profile = null) - * @method static float calculateShannonEntropy(string $string) - * @method static bool isCommonPattern(string $string, \Kirschbaum\Redactor\RedactorConfig $config) + * @method static \Kirschbaum\Redactor\RedactionResult inspect(mixed $content, ?string $profile = null, ?bool $mark = null, ?\Kirschbaum\Redactor\Detection\EntityFilter $entities = null) + * @method static mixed redactSafely(mixed $content, ?string $profile = null) + * @method static mixed detokenize(mixed $content) + * @method static bool registerSecret(string $value, string $entity = 'known_secret') + * @method static void registerOperator(string $name, \Kirschbaum\Redactor\Operators\Operator $operator) + * @method static void registerRecognizer(\Kirschbaum\Redactor\Recognition\Recognizer $recognizer) + * @method static void registerCustomStrategy(string $name, \Kirschbaum\Redactor\Strategies\Contracts\Strategy $strategy) + * @method static \Kirschbaum\Redactor\Operators\OperatorRegistry operators() + * @method static \Kirschbaum\Redactor\Recognition\RecognizerRegistry recognizers() + * @method static array validateProfiles() + * @method static array profiles() + * @method static bool hasProfile(string $profile) + * @method static array strategies(?string $profile = null) * * @see \Kirschbaum\Redactor\Redactor */ class Redactor extends Facade { + /** + * Get the registered name of the component. + */ protected static function getFacadeAccessor(): string { return \Kirschbaum\Redactor\Redactor::class; } + + /** + * Replace the bound redactor with a fake that records every call. + * + * It still redacts for real. Install it before the code under test + * resolves the redactor, such as before a log channel is first used, + * and assert afterwards with assertNeverEmitted() and friends. + */ + public static function fake(): RedactorFake + { + $fake = new RedactorFake; + + static::swap($fake); + + return $fake; + } } diff --git a/src/Findings/MatchFinding.php b/src/Findings/MatchFinding.php new file mode 100644 index 0000000..e9fd27b --- /dev/null +++ b/src/Findings/MatchFinding.php @@ -0,0 +1,72 @@ + + */ +final readonly class MatchFinding implements Arrayable, JsonSerializable +{ + public function __construct( + public string $rule, + public string $key = '', + public int $offset = 0, + public int $length = 0, + public string $matched = '', + /** What kind of thing was found; defaults to the rule that found it. */ + public ?string $entity = null, + /** How sure the detector was and why; null where certainty is not a question, such as a blocked key. */ + public ?Confidence $confidence = null, + ) {} + + /** + * Get the kind of thing that was found. + */ + public function entity(): string + { + return $this->entity ?? $this->rule; + } + + /** + * Get the finding as an array. + * + * The matched text is deliberately omitted. + * + * @return array + */ + public function toArray(): array + { + return [ + 'rule' => $this->rule, + 'entity' => $this->entity(), + 'key' => $this->key, + 'offset' => $this->offset, + 'length' => $this->length, + 'confidence' => $this->confidence?->score, + 'signals' => $this->confidence?->explain() ?? [], + ]; + } + + /** + * Convert the finding into something JSON serializable. + * + * @return array + */ + public function jsonSerialize(): array + { + return $this->toArray(); + } +} diff --git a/src/Http/Middleware/RedactResponse.php b/src/Http/Middleware/RedactResponse.php new file mode 100644 index 0000000..84c12ff --- /dev/null +++ b/src/Http/Middleware/RedactResponse.php @@ -0,0 +1,117 @@ +middleware('redact:observability'); + * + * JSON responses are redacted as data so structure and types survive; text is + * redacted as text; a stream is redacted as it streams, with a hold-back so + * nothing split across two chunks gets through; files pass through. The + * profile's `_redacted` markers are never written into a response. + * + * Fails closed: if the response cannot be redacted the client gets a 500 with + * none of the original body. Run `redactor:validate` at deploy time. + */ +class RedactResponse +{ + public function __construct( + protected Redactor $redactor, + ) {} + + public function handle(Request $request, Closure $next, ?string $profile = null): mixed + { + $response = $next($request); + + if (! $response instanceof Response || $response instanceof BinaryFileResponse) { + return $response; + } + + if ($response instanceof StreamedResponse) { + $callback = $response->getCallback(); + + if ($callback instanceof Closure) { + $response->setCallback((new StreamRedactor($this->redactor, $profile))->wrap($callback)); + } + + return $response; + } + + try { + return $this->redact($response, $profile); + } catch (Throwable $e) { + InternalLog::warning('Response could not be redacted; replaced with an error response', [ + 'profile' => $profile, + 'exception_type' => $e::class, + 'exception_message' => $e->getMessage(), + ]); + + return new JsonResponse(['message' => 'The response could not be redacted.'], 500); + } + } + + protected function redact(Response $response, ?string $profile): Response + { + if ($response instanceof JsonResponse) { + $data = $response->getData(true); + + return $response->setData($this->redactor->inspect($data, $profile, mark: false)->value); + } + + $content = $response->getContent(); + + if ($content === false || $content === '' || ! $this->isText($response)) { + return $response; + } + + $contentType = (string) $response->headers->get('Content-Type', ''); + + if (str_contains($contentType, 'json')) { + $decoded = json_decode($content, true); + + if (json_last_error() === JSON_ERROR_NONE) { + $redacted = $this->redactor->inspect($decoded, $profile, mark: false)->value; + $encoded = json_encode($redacted, JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE); + + if ($encoded !== false) { + $response->setContent($encoded); + + return $response; + } + } + } + + $redacted = $this->redactor->inspect($content, $profile, mark: false)->value; + + $response->setContent(is_string($redacted) ? $redacted : (string) json_encode($redacted)); + + return $response; + } + + protected function isText(Response $response): bool + { + $type = strtolower((string) $response->headers->get('Content-Type', 'text/html')); + + return $type === '' + || str_starts_with($type, 'text/') + || str_contains($type, 'json') + || str_contains($type, 'xml') + || str_contains($type, 'javascript') + || str_contains($type, 'x-www-form-urlencoded'); + } +} diff --git a/src/Logging/CustomLogTap.php b/src/Logging/CustomLogTap.php deleted file mode 100644 index 77a9434..0000000 --- a/src/Logging/CustomLogTap.php +++ /dev/null @@ -1,21 +0,0 @@ -getHandlers() as $handler) { - if ($handler instanceof FormattableHandlerInterface) { - $handler->setFormatter(new ReadactFormatter); - } - } - } -} diff --git a/src/Logging/ReadactFormatter.php b/src/Logging/ReadactFormatter.php deleted file mode 100644 index 7cbfa01..0000000 --- a/src/Logging/ReadactFormatter.php +++ /dev/null @@ -1,38 +0,0 @@ -message); - - // Format the main log line - $output = sprintf( - '[%s] %s.%s: %s', - $record->datetime->format('Y-m-d H:i:s.u'), - $record->channel, - $record->level->getName(), - is_string($message) ? $message : json_encode($message) - ); - - // Add sanitized context data if present - if (! empty($record->context)) { - $sanitizedContext = Redactor::redact($record->context); - $output .= ' '.json_encode($sanitizedContext, JSON_UNESCAPED_SLASHES); - } - - return $output."\n"; - } - - public function formatBatch(array $records): string - { - return $this->format($records[0]); - } -} diff --git a/src/Logging/RedactorFormatter.php b/src/Logging/RedactorFormatter.php new file mode 100644 index 0000000..9e30aab --- /dev/null +++ b/src/Logging/RedactorFormatter.php @@ -0,0 +1,98 @@ +redact($record); + + if ($this->inner instanceof FormatterInterface) { + // Monolog 3 types FormatterInterface::format() as mixed, since a formatter may not render a string... + $formatted = $this->inner->format($record); + + return is_string($formatted) ? $formatted : (string) json_encode($formatted); + } + + $output = sprintf( + '[%s] %s.%s: %s', + $record->datetime->format('Y-m-d H:i:s.u'), + $record->channel, + $record->level->getName(), + $record->message + ); + + if ($record->context !== []) { + $output .= ' '.json_encode($record->context, JSON_UNESCAPED_SLASHES); + } + + if ($record->extra !== []) { + $output .= ' '.json_encode($record->extra, JSON_UNESCAPED_SLASHES); + } + + return $output."\n"; + } + + /** + * @param array $records + */ + public function formatBatch(array $records): string + { + if ($this->inner instanceof FormatterInterface) { + $formatted = $this->inner->formatBatch(array_map( + $this->redact(...), + $records + )); + + return is_string($formatted) ? $formatted : (string) json_encode($formatted); + } + + // Every record must be rendered; returning format($records[0]) drops the rest in a batching handler... + $output = ''; + + foreach ($records as $record) { + $output .= $this->format($record); + } + + return $output; + } + + /** + * Redact the record using the same never-throw path as the processor. + */ + protected function redact(LogRecord $record): LogRecord + { + $message = Redactor::redactSafely($record->message); + $context = $record->context === [] ? [] : Redactor::redactSafely($record->context); + $extra = $record->extra === [] ? [] : Redactor::redactSafely($record->extra); + + return $record->with( + message: is_string($message) ? $message : (string) json_encode($message), + context: is_array($context) ? $context : ['redaction' => $context], + extra: is_array($extra) ? $extra : ['redaction' => $extra], + ); + } +} diff --git a/src/Logging/RedactorFormatterTap.php b/src/Logging/RedactorFormatterTap.php new file mode 100644 index 0000000..4473b6e --- /dev/null +++ b/src/Logging/RedactorFormatterTap.php @@ -0,0 +1,28 @@ +getHandlers() as $handler) { + if ($handler instanceof FormattableHandlerInterface) { + $handler->setFormatter(new RedactorFormatter); + } + } + } +} diff --git a/src/Logging/RedactorProcessor.php b/src/Logging/RedactorProcessor.php new file mode 100644 index 0000000..d96a0f6 --- /dev/null +++ b/src/Logging/RedactorProcessor.php @@ -0,0 +1,69 @@ + [ + * 'driver' => 'stack', + * 'channels' => ['single'], + * 'tap' => [\Kirschbaum\Redactor\Logging\RedactorTap::class], + * ], + */ +class RedactorProcessor implements ProcessorInterface +{ + public function __construct( + protected Redactor $redactor, + protected ?string $profile = null, + ) {} + + public function __invoke(LogRecord $record): LogRecord + { + // redactSafely(), never redact(): a throw inside the logging pipeline takes the channel down... + $message = $this->redactor->redactSafely($record->message, $this->profile); + + $context = $this->redactArray($record->context); + $extra = $this->redactArray($record->extra); + + return $record->with( + message: is_string($message) ? $message : (string) json_encode($message), + context: $context, + extra: $extra, + ); + } + + /** + * @param array $data + * @return array + */ + protected function redactArray(array $data): array + { + if ($data === []) { + return $data; + } + + $redacted = $this->redactor->redactSafely($data, $this->profile); + + if (is_array($redacted)) { + return $redacted; + } + + // redactSafely() failed closed and returned a marker string; keep the record shaped as Monolog expects... + return ['redaction' => $redacted]; + } +} diff --git a/src/Logging/RedactorTap.php b/src/Logging/RedactorTap.php new file mode 100644 index 0000000..bb8dd31 --- /dev/null +++ b/src/Logging/RedactorTap.php @@ -0,0 +1,39 @@ + [ + * 'driver' => 'single', + * 'path' => storage_path('logs/laravel.log'), + * 'tap' => [\Kirschbaum\Redactor\Logging\RedactorTap::class], + * ], + * + * Pass a profile name with the tap if the channel needs one other than the + * configured default: + * + * 'tap' => [\Kirschbaum\Redactor\Logging\RedactorTap::class.':strict'], + */ +class RedactorTap +{ + public function __invoke(Logger $logger, ?string $profile = null): void + { + $monolog = $logger->getLogger(); + + // Laravel types this as PSR-3; only Monolog takes processors... + if (! $monolog instanceof Monolog) { + return; + } + + $monolog->pushProcessor(new RedactorProcessor(Container::getInstance()->make(Redactor::class), $profile)); + } +} diff --git a/src/Mcp/McpResponseRedactor.php b/src/Mcp/McpResponseRedactor.php new file mode 100644 index 0000000..180a64d --- /dev/null +++ b/src/Mcp/McpResponseRedactor.php @@ -0,0 +1,159 @@ +|JsonRpcResponse $response + * @return iterable|JsonRpcResponse + */ + public function redact(iterable|JsonRpcResponse $response): iterable|JsonRpcResponse + { + if ($response instanceof JsonRpcResponse) { + return $this->redactOne($response); + } + + return $this->redactEach($response); + } + + /** + * @param iterable $responses + * @return Generator + */ + private function redactEach(iterable $responses): Generator + { + foreach ($responses as $response) { + yield $this->redactOne($response); + } + } + + private function redactOne(JsonRpcResponse $response): JsonRpcResponse + { + $content = $response->content; + + if (isset($content['result']) && is_array($content['result'])) { + $content['result'] = $this->redactResult($content['result']); + } + + if (isset($content['error']) && is_array($content['error']) && isset($content['error']['message']) && is_string($content['error']['message'])) { + $content['error']['message'] = $this->text($content['error']['message']); + } + + if (isset($content['params']) && is_array($content['params']) && isset($content['params']['content'])) { + // A streamed notification carrying content, as a tool yields... + $content['params'] = $this->redactResult($content['params']); + } + + $response->content = $content; + + return $response; + } + + /** + * @param array $result + * @return array + */ + private function redactResult(array $result): array + { + // tools/call and streamed tool output... + if (isset($result['content']) && is_array($result['content'])) { + $result['content'] = array_map($this->contentItem(...), $result['content']); + } + + if (isset($result['structuredContent']) && is_array($result['structuredContent'])) { + $result['structuredContent'] = $this->data($result['structuredContent']); + } + + // resources/read... + if (isset($result['contents']) && is_array($result['contents'])) { + $result['contents'] = array_map($this->contentItem(...), $result['contents']); + } + + // prompts/get... + if (isset($result['messages']) && is_array($result['messages'])) { + $result['messages'] = array_map(function ($message) { + if (is_array($message) && isset($message['content'])) { + $message['content'] = is_array($message['content']) && ! array_is_list($message['content']) + ? $this->contentItem($message['content']) + : (is_string($message['content']) ? $this->text($message['content']) : $message['content']); + } + + return $message; + }, $result['messages']); + } + + return $result; + } + + /** + * Redact a content block's text, leaving anything binary untouched. + */ + private function contentItem(mixed $item): mixed + { + if (! is_array($item)) { + return $item; + } + + if (isset($item['text']) && is_string($item['text'])) { + $item['text'] = $this->text($item['text']); + } + + if (isset($item['resource']) && is_array($item['resource']) && isset($item['resource']['text']) && is_string($item['resource']['text'])) { + $item['resource']['text'] = $this->text($item['resource']['text']); + } + + return $item; + } + + private function text(string $text): string + { + $out = $this->redactor->redactSafely($text, $this->profile); + + return is_string($out) ? $out : (string) json_encode($out); + } + + /** + * @param array $data + * @return array + */ + private function data(array $data): array + { + try { + // No markers: structured content has a schema the model was told about, and an unexpected key breaks it... + $out = $this->redactor->inspect($data, $this->profile, mark: false)->value; + } catch (\Throwable $e) { + InternalLog::warning('MCP structured content could not be redacted; replaced as a precaution', [ + 'profile' => $this->profile, + 'exception_type' => $e::class, + 'exception_message' => $e->getMessage(), + ]); + + return ['redaction' => 'failed']; + } + + return is_array($out) ? $out : ['redaction' => $out]; + } +} diff --git a/src/Mcp/RedactsResponses.php b/src/Mcp/RedactsResponses.php new file mode 100644 index 0000000..0db8d09 --- /dev/null +++ b/src/Mcp/RedactsResponses.php @@ -0,0 +1,52 @@ +|JsonRpcResponse + */ + protected function runMethodHandle(JsonRpcRequest $request, ServerContext $context): iterable|JsonRpcResponse + { + $response = parent::runMethodHandle($request, $context); + + return (new McpResponseRedactor(Container::getInstance()->make(Redactor::class), $this->redactionProfile()))->redact($response); + } +} diff --git a/src/Operators/HashOperator.php b/src/Operators/HashOperator.php new file mode 100644 index 0000000..c3da1bd --- /dev/null +++ b/src/Operators/HashOperator.php @@ -0,0 +1,40 @@ + [email:k4m9rp2xzq] + * + * The same value always yields the same token, so records stay countable and + * joinable, and the token is obviously not real data, which is what you want + * where a format-preserving surrogate could be mistaken for the genuine value. + */ +class HashOperator implements Operator +{ + /** + * Replace the span with a keyed token. + */ + public function apply(Detection $detection, OperatorContext $context): string + { + $pseudonymizer = $context->pseudonymizer(); + + if (! $pseudonymizer instanceof Pseudonymizer) { + // No key configured, so fail closed to a plain redaction rather than emit anything derived from the original... + return $context->replacement; + } + + $length = max(4, min(64, $context->intOption('length', 10))); + $token = $pseudonymizer->token($detection->entity, $detection->value, $length); + + return $context->boolOption('labelled', true) + ? sprintf('[%s:%s]', $detection->entity, $token) + : $token; + } +} diff --git a/src/Operators/MaskOperator.php b/src/Operators/MaskOperator.php new file mode 100644 index 0000000..94ba86a --- /dev/null +++ b/src/Operators/MaskOperator.php @@ -0,0 +1,23 @@ +stringOption('mask_character', '*'), 0, 1); + + return str_repeat($char, max(1, mb_strlen($detection->value))); + } +} diff --git a/src/Operators/NullifyOperator.php b/src/Operators/NullifyOperator.php new file mode 100644 index 0000000..00b55aa --- /dev/null +++ b/src/Operators/NullifyOperator.php @@ -0,0 +1,27 @@ + $options + * @param Pseudonymizer|Closure(): ?Pseudonymizer|null $pseudonymizer the pseudonymizer, or a resolver for one + */ + public function __construct( + public readonly string $replacement, + public readonly array $options = [], + private readonly Pseudonymizer|Closure|null $pseudonymizer = null, + ) {} + + /** + * Get the pseudonymizer, resolving it on first use. + * + * Only operators that pseudonymise pay for the key derivation. + */ + public function pseudonymizer(): ?Pseudonymizer + { + if ($this->isResolved) { + return $this->resolved; + } + + $this->isResolved = true; + $this->resolved = $this->pseudonymizer instanceof Closure + ? ($this->pseudonymizer)() + : $this->pseudonymizer; + + return $this->resolved; + } + + /** + * Get an integer option, or the default. + */ + public function intOption(string $key, int $default): int + { + $value = $this->options[$key] ?? null; + + return is_numeric($value) ? (int) $value : $default; + } + + /** + * Get a boolean option, or the default. + */ + public function boolOption(string $key, bool $default): bool + { + $value = $this->options[$key] ?? null; + + return is_bool($value) ? $value : $default; + } + + /** + * Get a non-empty string option, or the default. + */ + public function stringOption(string $key, string $default): string + { + $value = $this->options[$key] ?? null; + + return is_string($value) && $value !== '' ? $value : $default; + } +} diff --git a/src/Operators/OperatorRegistry.php b/src/Operators/OperatorRegistry.php new file mode 100644 index 0000000..0b2af0f --- /dev/null +++ b/src/Operators/OperatorRegistry.php @@ -0,0 +1,97 @@ + */ + private array $operators; + + /** + * Create a new operator registry instance. + */ + public function __construct(?SurrogateFactory $surrogates = null) + { + $this->operators = [ + self::REDACT => new RedactOperator, + self::MASK => new MaskOperator, + self::PARTIAL => new PartialOperator, + self::REMOVE => new RemoveOperator, + self::PRESERVE => new PreserveOperator, + self::HASH => new HashOperator, + self::SURROGATE => new SurrogateOperator($surrogates ?? new SurrogateFactory), + self::NULLIFY => new NullifyOperator, + ]; + } + + /** + * Register an operator under the given name. + */ + public function register(string $name, Operator $operator): void + { + $this->operators[$name] = $operator; + } + + /** + * Determine if an operator is registered under the given name. + */ + public function has(string $name): bool + { + return isset($this->operators[$name]); + } + + /** + * Get the operator registered under the given name. + * + * @throws ConfigurationException + */ + public function get(string $name): Operator + { + return $this->operators[$name] ?? throw new ConfigurationException(sprintf( + 'Unknown redaction operator [%s]. Available: %s.', + $name, + implode(', ', $this->names()) + )); + } + + /** + * Get the registered operator names. + * + * @return array + */ + public function names(): array + { + $names = array_keys($this->operators); + sort($names); + + return $names; + } +} diff --git a/src/Operators/OperatorSpec.php b/src/Operators/OperatorSpec.php new file mode 100644 index 0000000..19cc917 --- /dev/null +++ b/src/Operators/OperatorSpec.php @@ -0,0 +1,91 @@ + ['keep' => 4]] // name with options + * ['operator' => 'partial', 'keep' => 4] + */ +final readonly class OperatorSpec +{ + /** + * Create a new operator spec instance. + * + * @param array $options + */ + public function __construct( + public string $name, + public array $options = [], + ) {} + + /** + * Parse an operator definition from its configured form. + * + * @throws ConfigurationException + */ + public static function parse(mixed $definition, string $path): self + { + if ($definition instanceof self) { + return $definition; + } + + if (is_string($definition)) { + return new self($definition); + } + + if (! is_array($definition) || $definition === []) { + throw new ConfigurationException(sprintf( + 'Redactor config [%s] must name an operator.', + $path + )); + } + + if (isset($definition['operator']) && is_string($definition['operator'])) { + $options = $definition; + unset($options['operator']); + + return new self($definition['operator'], self::stringKeyed($options)); + } + + // A single name mapped to its options, as in ['partial' => ['keep' => 4]]... + $name = array_key_first($definition); + + if (! is_string($name)) { + throw new ConfigurationException(sprintf( + 'Redactor config [%s] must name an operator.', + $path + )); + } + + $options = $definition[$name]; + + return new self($name, is_array($options) ? self::stringKeyed($options) : []); + } + + /** + * Cast every option key to a string. + * + * @param array $options + * @return array + */ + private static function stringKeyed(array $options): array + { + $out = []; + + foreach ($options as $key => $value) { + $out[(string) $key] = $value; + } + + return $out; + } +} diff --git a/src/Operators/PartialOperator.php b/src/Operators/PartialOperator.php new file mode 100644 index 0000000..4e94224 --- /dev/null +++ b/src/Operators/PartialOperator.php @@ -0,0 +1,33 @@ +intOption('keep', 4)); + $char = mb_substr($context->stringOption('mask_character', '*'), 0, 1); + $length = mb_strlen($detection->value); + + if ($length <= $keep) { + // Too short to reveal any of it without revealing all of it... + return str_repeat($char, max(1, $length)); + } + + return str_repeat($char, $length - $keep).mb_substr($detection->value, -$keep); + } +} diff --git a/src/Operators/PreserveOperator.php b/src/Operators/PreserveOperator.php new file mode 100644 index 0000000..d974ea7 --- /dev/null +++ b/src/Operators/PreserveOperator.php @@ -0,0 +1,25 @@ +value; + } +} diff --git a/src/Operators/RedactOperator.php b/src/Operators/RedactOperator.php new file mode 100644 index 0000000..d5726fb --- /dev/null +++ b/src/Operators/RedactOperator.php @@ -0,0 +1,21 @@ +replacement; + } +} diff --git a/src/Operators/RedactionPolicy.php b/src/Operators/RedactionPolicy.php new file mode 100644 index 0000000..4fa99ea --- /dev/null +++ b/src/Operators/RedactionPolicy.php @@ -0,0 +1,52 @@ + $byEntity keyed by entity, plus 'default' + */ + public function __construct( + private array $byEntity = [], + private OperatorSpec $default = new OperatorSpec(OperatorRegistry::REDACT), + ) {} + + /** + * Resolve the operator that applies to the given detection. + */ + public function operatorFor(Detection $detection, ?OperatorSpec $atLocation = null): OperatorSpec + { + if ($atLocation instanceof OperatorSpec) { + return $atLocation; + } + + if (isset($this->byEntity[$detection->entity])) { + return $this->byEntity[$detection->entity]; + } + + // Only a rule that actually chose an operator outranks the profile default, + // since treating a rule's implied default as a choice would make + // `operators.default` unreachable for anything found by a pattern... + if ($detection->operator instanceof OperatorSpec) { + return $detection->operator; + } + + return $this->byEntity['default'] ?? $this->default; + } +} diff --git a/src/Operators/RemoveOperator.php b/src/Operators/RemoveOperator.php new file mode 100644 index 0000000..d782360 --- /dev/null +++ b/src/Operators/RemoveOperator.php @@ -0,0 +1,21 @@ + u_7f3ac9@customer.com + * 4111 1111 1111 1111 -> 4111 1193 7420 8846 + * sk_live_4eC39HqLyj -> sk_live_9mB71TzKnQ + * + * This is what keeps a redacted log usable. "[REDACTED]" collapses every + * distinct value into one, which destroys counts, joins and traces; a stable + * surrogate preserves all three while leaking none of the original. + */ +class SurrogateOperator implements Operator +{ + /** + * Create a new surrogate operator instance. + */ + public function __construct( + private readonly SurrogateFactory $surrogates = new SurrogateFactory, + ) {} + + /** + * Replace the span with a surrogate of the same shape. + */ + public function apply(Detection $detection, OperatorContext $context): string + { + $pseudonymizer = $context->pseudonymizer(); + + if (! $pseudonymizer instanceof Pseudonymizer) { + // Without a key there is no stable mapping to produce, and an unstable one would look joinable and silently not be... + return $context->replacement; + } + + return $this->surrogates->generate( + $detection->entity, + $detection->value, + $pseudonymizer->random($detection->entity, $detection->value), + $context->options, + ); + } +} diff --git a/src/Operators/Surrogates/CharacterClassSurrogate.php b/src/Operators/Surrogates/CharacterClassSurrogate.php new file mode 100644 index 0000000..178991f --- /dev/null +++ b/src/Operators/Surrogates/CharacterClassSurrogate.php @@ -0,0 +1,65 @@ + sk_live_9mB71TzKnQxPvfhs + * +1 (555) 867-5309 -> +7 (204) 331-8874 + * + * Length, separators, capitalisation and digit positions all survive, so + * anything parsing the value keeps parsing it, and nothing of the original + * survives except its shape. It works on any value, which is what makes it + * the fallback for entities nobody wrote a generator for. + */ +class CharacterClassSurrogate implements SurrogateGenerator +{ + private const string LOWER = 'abcdefghijklmnopqrstuvwxyz'; + + private const string UPPER = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'; + + private const string DIGITS = '0123456789'; + + /** + * Determine if the generator can stand in for the given value. + */ + public function supports(string $entity, string $value): bool + { + return true; + } + + /** + * Generate a surrogate for the given value. + * + * @param array $options + */ + public function generate(string $value, DeterministicRandom $random, array $options = []): string + { + // A prefix such as "sk_live_" tells an on-call engineer which credential leaked, which is the point of the log line... + $keep = $options['preserve_prefix'] ?? 0; + $keep = is_int($keep) ? max(0, min($keep, strlen($value))) : 0; + + $out = substr($value, 0, $keep); + + $length = strlen($value); + + for ($i = $keep; $i < $length; $i++) { + $char = $value[$i]; + + $out .= match (true) { + $char >= 'a' && $char <= 'z' => $random->pick(self::LOWER), + $char >= 'A' && $char <= 'Z' => $random->pick(self::UPPER), + $char >= '0' && $char <= '9' => $random->pick(self::DIGITS), + // Separators, punctuation and anything multibyte pass through, since they carry the structure, not the secret... + default => $char, + }; + } + + return $out; + } +} diff --git a/src/Operators/Surrogates/CreditCardSurrogate.php b/src/Operators/Surrogates/CreditCardSurrogate.php new file mode 100644 index 0000000..a89bb43 --- /dev/null +++ b/src/Operators/Surrogates/CreditCardSurrogate.php @@ -0,0 +1,119 @@ + 4111 1193 7420 8846 + * + * Length, grouping and the issuer prefix survive; the account number does not. + * The check digit is recomputed so the result validates, since a fixture or a + * replayed request carrying an invalid card fails at a different layer than + * the one under test. The BIN is kept by default: it identifies the issuer and + * card type, which fraud and finance teams aggregate on, and is not specific + * to a cardholder. + */ +class CreditCardSurrogate implements SurrogateGenerator +{ + private const int DEFAULT_BIN_LENGTH = 6; + + /** + * Determine if the value is a card number or is flagged as one. + */ + public function supports(string $entity, string $value): bool + { + if ($entity === 'credit_card') { + return true; + } + + $digits = preg_replace('/\D/', '', $value) ?? ''; + + return strlen($digits) >= 12 && strlen($digits) <= 19; + } + + /** + * Generate a Luhn-valid surrogate for the given card number. + * + * @param array $options + */ + public function generate(string $value, DeterministicRandom $random, array $options = []): string + { + $digits = preg_replace('/\D/', '', $value) ?? ''; + $count = strlen($digits); + + if ($count < 2) { + return $value; + } + + $binLength = $options['preserve_bin'] ?? self::DEFAULT_BIN_LENGTH; + $binLength = is_int($binLength) ? max(0, min($binLength, $count - 2)) : self::DEFAULT_BIN_LENGTH; + $binLength = min($binLength, $count - 2); + + $generated = substr($digits, 0, $binLength); + + // Everything between the BIN and the check digit is replaced... + for ($i = $binLength; $i < $count - 1; $i++) { + $generated .= $random->digit(); + } + + $generated .= $this->checkDigit($generated); + + return $this->reapplyFormatting($value, $generated); + } + + /** + * Get the digit that makes a Luhn sum land on a multiple of ten. + */ + private function checkDigit(string $withoutCheck): string + { + $sum = 0; + // The check digit sits in an undoubled position... + $double = true; + + for ($i = strlen($withoutCheck) - 1; $i >= 0; $i--) { + $digit = (int) $withoutCheck[$i]; + + if ($double) { + $digit *= 2; + if ($digit > 9) { + $digit -= 9; + } + } + + $sum += $digit; + $double = ! $double; + } + + return (string) ((10 - ($sum % 10)) % 10); + } + + /** + * Put the original spaces and dashes back where they were. + */ + private function reapplyFormatting(string $original, string $digits): string + { + $out = ''; + $index = 0; + $length = strlen($original); + + for ($i = 0; $i < $length; $i++) { + $char = $original[$i]; + + if ($char >= '0' && $char <= '9') { + $out .= $digits[$index] ?? $char; + $index++; + + continue; + } + + $out .= $char; + } + + return $out; + } +} diff --git a/src/Operators/Surrogates/EmailSurrogate.php b/src/Operators/Surrogates/EmailSurrogate.php new file mode 100644 index 0000000..77cb780 --- /dev/null +++ b/src/Operators/Surrogates/EmailSurrogate.php @@ -0,0 +1,54 @@ + u_7f3ac9@customer.com (domain kept) + * alice@customer.com -> u_7f3ac9@example.invalid (domain replaced) + * + * Keeping the domain preserves the analysis people run on logs, which tenant, + * which provider, how many distinct users at one company, while losing the + * individual. Replacing it uses .invalid, which RFC 2606 guarantees can never + * resolve, so a surrogate that escapes into a mail queue bounces instead of + * reaching a stranger. + */ +class EmailSurrogate implements SurrogateGenerator +{ + /** + * Determine if the value is an email address or is flagged as one. + */ + public function supports(string $entity, string $value): bool + { + return $entity === 'email' || (str_contains($value, '@') && substr_count($value, '@') === 1); + } + + /** + * Generate a surrogate address for the given one. + * + * @param array $options + */ + public function generate(string $value, DeterministicRandom $random, array $options = []): string + { + $at = strrpos($value, '@'); + + if ($at === false) { + return 'u_'.$random->token(6).'@example.invalid'; + } + + // Normalised rather than raw, since the seed already lowercases and trims and a + // verbatim domain would give "Alice@Customer.COM" a different surrogate from + // "alice@customer.com", silently double-counting one user... + $domain = strtolower(trim(substr($value, $at + 1))); + $preserveDomain = ($options['preserve_domain'] ?? true) === true; + + $local = 'u_'.$random->token(6); + + return $local.'@'.($preserveDomain && $domain !== '' ? $domain : 'example.invalid'); + } +} diff --git a/src/Operators/Surrogates/SurrogateFactory.php b/src/Operators/Surrogates/SurrogateFactory.php new file mode 100644 index 0000000..ed79eae --- /dev/null +++ b/src/Operators/Surrogates/SurrogateFactory.php @@ -0,0 +1,65 @@ + */ + private array $generators; + + /** Supports everything, so it answers whenever nothing more specific does. */ + private readonly CharacterClassSurrogate $fallback; + + /** + * Create a new surrogate factory instance. + * + * @param array $custom + */ + public function __construct(array $custom = []) + { + $this->generators = [ + ...$custom, + new EmailSurrogate, + new CreditCardSurrogate, + ]; + $this->fallback = new CharacterClassSurrogate; + } + + /** + * Register a generator ahead of the built-in ones. + */ + public function register(SurrogateGenerator $generator): void + { + array_unshift($this->generators, $generator); + } + + /** + * Generate a surrogate using the first generator that supports the value. + * + * @param array $options + */ + public function generate(string $entity, string $value, DeterministicRandom $random, array $options = []): string + { + foreach ($this->generators as $generator) { + if ($generator->supports($entity, $value)) { + return $generator->generate($value, $random, $options); + } + } + + // Nothing more specific claimed it, so keep its shape and nothing else... + return $this->fallback->generate($value, $random, $options); + } +} diff --git a/src/Operators/Surrogates/SurrogateGenerator.php b/src/Operators/Surrogates/SurrogateGenerator.php new file mode 100644 index 0000000..17f4724 --- /dev/null +++ b/src/Operators/Surrogates/SurrogateGenerator.php @@ -0,0 +1,31 @@ + $options + */ + public function generate(string $value, DeterministicRandom $random, array $options = []): string; +} diff --git a/src/Path/PathCursor.php b/src/Path/PathCursor.php new file mode 100644 index 0000000..175440b --- /dev/null +++ b/src/Path/PathCursor.php @@ -0,0 +1,53 @@ + $states + */ + public function __construct( + private PathTrie $trie, + private array $states = [], + ) {} + + /** + * Descend one segment and get the cursor for the child. + */ + public function descend(string $segment): self + { + return $this->states === [] + ? $this + : new self($this->trie, $this->trie->advance($this->states, $segment)); + } + + /** + * Get the winning path rule at the current position, if any. + */ + public function match(): ?PathMatch + { + return $this->states === [] ? null : $this->trie->match($this->states); + } + + /** + * Determine if this cursor can no longer lead anywhere. + */ + public function isExhausted(): bool + { + return $this->states === []; + } +} diff --git a/src/Path/PathMatch.php b/src/Path/PathMatch.php new file mode 100644 index 0000000..f1e9c47 --- /dev/null +++ b/src/Path/PathMatch.php @@ -0,0 +1,26 @@ + $segments + */ + private function __construct( + public string $source, + public array $segments, + public int $specificity, + ) {} + + /** + * Parse a dotted path pattern. + * + * @throws ConfigurationException + */ + public static function parse(string $pattern): self + { + $normalised = self::normalise($pattern); + + if ($normalised === []) { + throw new ConfigurationException(sprintf( + 'Redactor path pattern [%s] is empty.', + $pattern + )); + } + + return new self($pattern, $normalised, self::score($normalised)); + } + + /** + * Split a pattern into segments, turning list syntax into ordinary ones. + * + * `users[*].email` and `users.*.email` describe the same place; accepting + * both means nobody has to remember which spelling this library chose. + * + * @return array + */ + private static function normalise(string $pattern): array + { + // users[*] becomes users.* and items[0] becomes items.0... + $expanded = preg_replace('/\[([^\]]*)\]/', '.$1', $pattern) ?? $pattern; + $expanded = str_replace('..', '.', $expanded); + + $segments = []; + + foreach (explode('.', $expanded) as $segment) { + $segment = trim($segment); + + if ($segment === '') { + // An empty `[]` means "any index", and a stray dot is noise... + continue; + } + + $segments[] = strtolower($segment); + } + + return $segments; + } + + /** + * Score how specific a pattern is, for resolving overlaps. + * + * A literal segment says the most, a single-level wildcard less, and a + * deep wildcard least, so `request.headers.authorization` beats + * `request.headers.*`, which beats `**.authorization`. Without an ordering + * the winner would depend on config order. + * + * @param array $segments + */ + private static function score(array $segments): int + { + $score = 0; + + foreach ($segments as $segment) { + $score += match ($segment) { + self::DEEP => 1, + self::ANY => 2, + default => 3, + }; + } + + return $score; + } +} diff --git a/src/Path/PathTrie.php b/src/Path/PathTrie.php new file mode 100644 index 0000000..ace60b6 --- /dev/null +++ b/src/Path/PathTrie.php @@ -0,0 +1,243 @@ +> literal segment => child node */ + private array $children = [self::ROOT => []]; + + /** @var array the `*` child of each node */ + private array $any = [self::ROOT => null]; + + /** @var array the `**` child of each node */ + private array $deep = [self::ROOT => null]; + + /** @var array whether a node is itself a `**` node */ + private array $isDeep = [self::ROOT => false]; + + /** @var array */ + private array $terminal = [self::ROOT => null]; + + private int $nextNode = 1; + + private bool $empty = true; + + /** + * The compiled tries, keyed by the rule set that produced them. + * + * The profile config is resolved on every redaction, so without this the + * trie would be rebuilt per call: measured at 0.23ms per call for 200 + * rules, several times the cost of the redaction itself. + * + * @var array + */ + private static array $memo = []; + + /** + * Compile a set of path rules, reusing the trie for identical sets. + * + * @param array $rules path pattern => operator + */ + public static function compile(array $rules): self + { + if ($rules === []) { + return new self; + } + + $cacheKey = self::cacheKey($rules); + + if (isset(self::$memo[$cacheKey])) { + return self::$memo[$cacheKey]; + } + + $trie = new self; + + foreach ($rules as $pattern => $spec) { + // PHP turns a purely numeric array key into an int, so a rule targeting a + // list index arrives here as an integer and has to be put back... + $trie->add(PathPattern::parse((string) $pattern), $spec); + } + + return self::$memo[$cacheKey] = $trie; + } + + /** + * Identify a rule set by its patterns and what they do. + * + * Both halves matter: changing an operator without changing a pattern + * must still produce a different trie, or a config change would silently + * fail to take effect and turn a security setting into a no-op. + * + * @param array $rules + */ + private static function cacheKey(array $rules): string + { + $parts = []; + + foreach ($rules as $pattern => $spec) { + $options = json_encode($spec->options); + $parts[] = $pattern.'>'.$spec->name.'>'.($options === false ? '' : $options); + } + + return implode("\0", $parts); + } + + /** + * Determine if the trie has no rules. + */ + public function isEmpty(): bool + { + return $this->empty; + } + + /** + * Create a cursor positioned at the root. + */ + public function cursor(): PathCursor + { + return new PathCursor($this, $this->empty ? [] : [self::ROOT]); + } + + /** + * Add a pattern to the trie. + */ + private function add(PathPattern $pattern, OperatorSpec $spec): void + { + $this->empty = false; + + $node = self::ROOT; + + foreach ($pattern->segments as $segment) { + $node = match ($segment) { + PathPattern::DEEP => $this->deep[$node] ??= $this->createNode(isDeep: true), + PathPattern::ANY => $this->any[$node] ??= $this->createNode(), + default => $this->children[$node][$segment] ??= $this->createNode(), + }; + } + + $existing = $this->terminal[$node]; + + // Two patterns reaching the same node keep the more specific one, so the winner does not depend on declaration order... + if ($existing === null || $pattern->specificity >= $existing['specificity']) { + $this->terminal[$node] = [ + 'spec' => $spec, + 'specificity' => $pattern->specificity, + 'source' => $pattern->source, + ]; + } + } + + /** + * Create a new node and get its id. + */ + private function createNode(bool $isDeep = false): int + { + $id = $this->nextNode++; + + $this->children[$id] = []; + $this->any[$id] = null; + $this->deep[$id] = null; + $this->isDeep[$id] = $isDeep; + $this->terminal[$id] = null; + + return $id; + } + + /** + * Advance a set of states by one path segment. + * + * @param array $states + * @return array + */ + public function advance(array $states, string $segment): array + { + if ($states === []) { + return []; + } + + $segment = strtolower($segment); + $next = []; + + foreach ($states as $state) { + // A `**` node absorbs this segment and stays in play for the next... + if ($this->isDeep[$state]) { + $next[$state] = $state; + } + + $this->step($state, $segment, $next); + + // A `**` child can absorb this segment, or match zero segments and let what follows it match here instead... + $deep = $this->deep[$state]; + + if ($deep !== null) { + $next[$deep] = $deep; + $this->step($deep, $segment, $next); + } + } + + return $next; + } + + /** + * Follow the literal and `*` edges from a state. + * + * @param array $next + */ + private function step(int $state, string $segment, array &$next): void + { + $literal = $this->children[$state][$segment] ?? null; + + if ($literal !== null) { + $next[$literal] = $literal; + } + + $any = $this->any[$state]; + + if ($any !== null) { + $next[$any] = $any; + } + } + + /** + * Get the winning operator among a set of states, if any of them is terminal. + * + * @param array $states + */ + public function match(array $states): ?PathMatch + { + $best = null; + + foreach ($states as $state) { + $terminal = $this->terminal[$state] ?? null; + + if ($terminal === null) { + continue; + } + + if ($best === null || $terminal['specificity'] > $best['specificity']) { + $best = $terminal; + } + } + + return $best === null + ? null + : new PathMatch($best['spec'], $best['source'], $best['specificity']); + } +} diff --git a/src/Patterns/PatternRule.php b/src/Patterns/PatternRule.php new file mode 100644 index 0000000..09c0431 --- /dev/null +++ b/src/Patterns/PatternRule.php @@ -0,0 +1,274 @@ + '/[^@\s]+@[^@\s]+/', + * + * or as a full rule: + * + * 'credit_card' => [ + * 'pattern' => '/\b(?:\d[ -]*?){13,16}\b/', + * 'mode' => 'partial', + * 'keep' => 4, + * ], + */ +final readonly class PatternRule +{ + /** Replace just the matched text with the replacement string. */ + public const string MODE_REPLACE = 'replace'; + + /** Replace each matched character with a mask character, preserving length. */ + public const string MODE_MASK = 'mask'; + + /** Keep the last N characters of the match and mask the rest. */ + public const string MODE_PARTIAL = 'partial'; + + /** Delete the matched text entirely. */ + public const string MODE_REMOVE = 'remove'; + + /** Replace the whole value, not just the match. The pre-1.0 behaviour. */ + public const string MODE_FULL = 'full'; + + /** @var array */ + public const array MODES = [ + self::MODE_REPLACE, + self::MODE_MASK, + self::MODE_PARTIAL, + self::MODE_REMOVE, + self::MODE_FULL, + ]; + + /** + * Create a new pattern rule instance. + * + * @param array $keywords + * @param array $samples + * @param array $counterSamples + */ + public function __construct( + public string $name, + public string $pattern, + public string $mode = self::MODE_REPLACE, + public int $keep = 4, + public string $maskCharacter = '*', + /** The capture group holding the secret, or 0 for the whole match, so a labelled value keeps its label. */ + public int $capture = 0, + /** The structural check the matched text must pass, or null when shape is enough. */ + public ?string $validator = null, + /** The kind of thing this rule finds, defaulting to the rule name. */ + public ?string $entity = null, + /** How much to trust a bare match, before validators and context adjust it. */ + public float $confidence = Confidence::MEDIUM, + /** What to do with what it finds, or null to let the profile decide. */ + public ?OperatorSpec $operator = null, + /** Lowercased literals at least one of which must appear in the subject before the pattern is tried. */ + public array $keywords = [], + /** Matches this rule alone should let through, scoped to the rule unlike the profile allowlist. */ + public ?AllowList $allow = null, + /** The shortest text this pattern can match, in bytes; never above the true minimum or matches are missed. */ + public int $minLength = 1, + /** Texts this rule must detect something in, checked by redactor:validate. */ + public array $samples = [], + /** Texts this rule must not detect anything in. */ + public array $counterSamples = [], + ) {} + + /** + * Get the entity this rule detects. + */ + public function entity(): string + { + return $this->entity ?? $this->name; + } + + /** + * Determine if this rule actually asked for a particular operator. + * + * `mode` defaults to replace, so operatorSpec() can always produce + * something, which is not the same as the rule having chosen it. Without + * this distinction a rule with no preference would still outrank the + * profile's `operators.default`, making that setting unreachable. + */ + public function hasExplicitOperator(): bool + { + return $this->operator instanceof OperatorSpec || $this->mode !== self::MODE_REPLACE; + } + + /** + * Get the operator this rule asks for, translating the legacy `mode` when none is set. + */ + public function operatorSpec(): OperatorSpec + { + if ($this->operator instanceof OperatorSpec) { + return $this->operator; + } + + return new OperatorSpec( + match ($this->mode) { + self::MODE_MASK => OperatorRegistry::MASK, + self::MODE_PARTIAL => OperatorRegistry::PARTIAL, + self::MODE_REMOVE => OperatorRegistry::REMOVE, + default => OperatorRegistry::REDACT, + }, + ['keep' => $this->keep, 'mask_character' => $this->maskCharacter], + ); + } + + /** + * Build a rule from its configured form, or return null if unusable. + * + * An uncompilable pattern is dropped rather than fatal; a malformed rule, + * such as a bad mode or a missing pattern, is a config error and throws. + * + * @throws ConfigurationException + */ + public static function fromConfig(string $name, mixed $definition, string $path): ?self + { + if (is_string($definition)) { + return Pcre::isValidPattern($definition) + ? new self(name: $name, pattern: $definition) + : null; + } + + if (! is_array($definition)) { + return null; + } + + $pattern = $definition['pattern'] ?? null; + + // A dictionary rule compiles a list of words into one alternation, for names no regex could express... + if ($pattern === null && isset($definition['words'])) { + $words = array_values(array_filter( + ConfigValue::stringList($definition['words'], $path.'.words'), + fn (string $word): bool => trim($word) !== '' + )); + + if ($words === []) { + throw new ConfigurationException(sprintf( + 'Redactor config [%s] lists no words.', + $path + )); + } + + usort($words, fn (string $a, string $b): int => strlen($b) <=> strlen($a)); + + $pattern = '/(? preg_quote(trim($word), '/'), $words)) + .')(?![\p{L}\p{N}])/iu'; + } + + if (! is_string($pattern)) { + throw new ConfigurationException(sprintf( + 'Redactor config [%s] must define a "pattern" string or a "words" list.', + $path + )); + } + + if (! Pcre::isValidPattern($pattern)) { + return null; + } + + $mode = ConfigValue::enum($definition['mode'] ?? self::MODE_REPLACE, self::MODES, self::MODE_REPLACE, $path.'.mode'); + $keep = ConfigValue::positiveInt($definition['keep'] ?? 4, 4, $path.'.keep'); + $maskCharacter = ConfigValue::string($definition['mask_character'] ?? '*', '*', $path.'.mask_character'); + $validator = $definition['validator'] ?? null; + + if ($validator !== null) { + $validator = ConfigValue::string($validator, Validator::LUHN, $path.'.validator'); + + if (! Validator::exists($validator)) { + throw new ConfigurationException(sprintf( + 'Redactor config [%s.validator] names an unknown validator [%s]. Known: %s.', + $path, + $validator, + implode(', ', Validator::NAMES) + )); + } + } + + $entity = $definition['entity'] ?? null; + $entity = is_string($entity) && $entity !== '' ? $entity : null; + + $confidence = $definition['confidence'] ?? Confidence::MEDIUM; + $confidence = is_numeric($confidence) + ? max(0.0, min(1.0, (float) $confidence)) + : Confidence::MEDIUM; + + $operator = isset($definition['operator']) + ? OperatorSpec::parse($definition['operator'], $path.'.operator') + : null; + + $capture = $definition['capture'] ?? 0; + $capture = $capture === 0 || $capture === '0' + ? 0 + : ConfigValue::positiveInt($capture, 0, $path.'.capture'); + + $keywords = array_values(array_filter(array_map( + strtolower(...), + ConfigValue::stringList($definition['keywords'] ?? [], $path.'.keywords') + ), fn (string $keyword): bool => $keyword !== '')); + + $allow = ConfigValue::stringList($definition['allow'] ?? [], $path.'.allow'); + + $minLength = ConfigValue::positiveInt($definition['min_length'] ?? 1, 1, $path.'.min_length'); + $samples = ConfigValue::stringList($definition['samples'] ?? [], $path.'.samples'); + $counterSamples = ConfigValue::stringList($definition['counter_samples'] ?? [], $path.'.counter_samples'); + + if ($maskCharacter === '') { + $maskCharacter = '*'; + } + + return new self( + name: $name, + pattern: $pattern, + mode: $mode, + keep: $keep, + maskCharacter: mb_substr($maskCharacter, 0, 1), + capture: $capture, + validator: $validator, + entity: $entity, + confidence: $confidence, + operator: $operator, + keywords: $keywords, + allow: $allow === [] ? null : AllowList::for($allow), + minLength: $minLength, + samples: $samples, + counterSamples: $counterSamples, + ); + } + + /** + * Determine if the matched text passes this rule's allow list and structural check. + */ + public function accepts(string $match): bool + { + if ($this->allow instanceof AllowList && $this->allow->allows($match)) { + return false; + } + + return $this->validator === null || Validator::passes($this->validator, $match); + } + + /** + * Determine if this rule replaces the entire value rather than the match. + */ + public function replacesWholeValue(): bool + { + return $this->mode === self::MODE_FULL; + } +} diff --git a/src/Patterns/Validator.php b/src/Patterns/Validator.php new file mode 100644 index 0000000..feaa3e3 --- /dev/null +++ b/src/Patterns/Validator.php @@ -0,0 +1,592 @@ + */ + public const NAMES = [ + self::LUHN, self::IBAN, self::SSN, self::NHS, self::BSN, self::STEUER_ID, self::NIR, self::DNI, + self::CODICE_FISCALE, self::BELGIAN_NATIONAL_NUMBER, self::PERSONNUMMER, self::FODSELSNUMMER, + self::SIN, self::TFN, self::VAT, + ]; + + /** @var array */ + protected static array $custom = []; + + /** + * Register a validator of your own, usable from config by name. + * + * @param callable(string): bool $check + */ + public static function extend(string $name, callable $check): void + { + static::$custom[$name] = $check; + } + + /** + * Determine if a validator of this name exists. + */ + public static function exists(string $name): bool + { + return isset(static::$custom[$name]) || in_array($name, self::NAMES, true); + } + + /** + * Determine if a value passes the named validator. + */ + public static function passes(string $name, string $value): bool + { + if (isset(static::$custom[$name])) { + return (bool) (static::$custom[$name])($value); + } + + return match ($name) { + self::LUHN => self::luhn($value), + self::IBAN => self::iban($value), + self::SSN => self::ssn($value), + self::NHS => self::nhs($value), + self::BSN => self::bsn($value), + self::STEUER_ID => self::steuerId($value), + self::NIR => self::nir($value), + self::DNI => self::dni($value), + self::CODICE_FISCALE => self::codiceFiscale($value), + self::BELGIAN_NATIONAL_NUMBER => self::belgianNationalNumber($value), + self::PERSONNUMMER => self::personnummer($value), + self::FODSELSNUMMER => self::fodselsnummer($value), + self::SIN => self::sin($value), + self::TFN => self::tfn($value), + self::VAT => self::vat($value), + // An unknown validator cannot be evaluated, so it must not veto a match and silently disable the rule... + default => true, + }; + } + + /** + * Determine if a value passes the Luhn check used by payment cards and IMEIs. + */ + public static function luhn(string $value): bool + { + $digits = self::digits($value); + $length = strlen($digits); + + if ($length < 12 || $length > 19) { + return false; + } + + return self::luhnSum($digits) % 10 === 0; + } + + /** + * Determine if a value passes the ISO 13616 mod-97 check. + */ + public static function iban(string $value): bool + { + $iban = strtoupper(preg_replace('/[^A-Za-z0-9]/', '', $value) ?? ''); + + if (strlen($iban) < 15 || strlen($iban) > 34) { + return false; + } + + if (preg_match('/^[A-Z]{2}\d{2}[A-Z0-9]+$/', $iban) !== 1) { + return false; + } + + return self::mod97(substr($iban, 4).substr($iban, 0, 4)) === 1; + } + + /** + * Determine if a value follows the US Social Security number allocation rules. + * + * Area 000, 666 and 900-999 have never been issued, and neither group 00 + * nor serial 0000 exists. Rejecting them removes most of the dates, phone + * fragments and sequence numbers that match the SSN shape. + */ + public static function ssn(string $value): bool + { + $digits = self::digits($value); + + if (strlen($digits) !== 9) { + return false; + } + + $area = (int) substr($digits, 0, 3); + $group = (int) substr($digits, 3, 2); + $serial = (int) substr($digits, 5, 4); + + if ($area === 0 || $area === 666 || $area >= 900) { + return false; + } + + return $group !== 0 && $serial !== 0; + } + + /** + * Determine if a value is a valid NHS number (ten digits, mod-11 check digit). + */ + public static function nhs(string $value): bool + { + $digits = self::digits($value); + + if (strlen($digits) !== 10) { + return false; + } + + $sum = 0; + + for ($i = 0; $i < 9; $i++) { + $sum += (int) $digits[$i] * (10 - $i); + } + + $check = 11 - ($sum % 11); + + if ($check === 11) { + $check = 0; + } + + return $check !== 10 && $check === (int) $digits[9]; + } + + /** + * Determine if a value is a valid Dutch citizen service number (the eleven-proof). + */ + public static function bsn(string $value): bool + { + $digits = self::digits($value); + + if (strlen($digits) !== 9) { + return false; + } + + $sum = 0; + + for ($i = 0; $i < 8; $i++) { + $sum += (int) $digits[$i] * (9 - $i); + } + + $sum -= (int) $digits[8]; + + return $sum % 11 === 0; + } + + /** + * Determine if a value is a valid German tax identification number. + * + * Eleven digits, not starting with zero, exactly one digit repeated among + * the first ten, and a check digit computed by the ISO 7064 mod 11,10 scheme. + */ + public static function steuerId(string $value): bool + { + $digits = self::digits($value); + + if (preg_match('/^[1-9]\d{10}$/', $digits) !== 1) { + return false; + } + + $counts = array_count_values(str_split(substr($digits, 0, 10))); + $repeated = array_filter($counts, fn (int $n): bool => $n > 1); + + if (count($repeated) !== 1 || max($repeated) > 3) { + return false; + } + + $product = 10; + + for ($i = 0; $i < 10; $i++) { + $sum = ((int) $digits[$i] + $product) % 10; + + if ($sum === 0) { + $sum = 10; + } + + $product = ($sum * 2) % 11; + } + + $check = 11 - $product; + + if ($check === 10) { + $check = 0; + } + + return $check === (int) $digits[10]; + } + + /** + * Determine if a value is a valid French social security number (NIR with its key). + */ + public static function nir(string $value): bool + { + $value = strtoupper(preg_replace('/\s/', '', $value) ?? ''); + + if (preg_match('/^[12]\d{2}(0[1-9]|1[0-2]|[2-9]\d)(\d{2}|2A|2B)\d{6}\d{2}$/', $value) !== 1) { + return false; + } + + $number = str_replace(['2A', '2B'], ['19', '18'], substr($value, 0, 13)); + $key = (int) substr($value, 13, 2); + + return 97 - self::mod97($number) === $key; + } + + /** + * Determine if a value is a valid Spanish DNI or NIE (the letter is a mod-23 check). + */ + public static function dni(string $value): bool + { + $value = strtoupper(preg_replace('/[\s-]/', '', $value) ?? ''); + $letters = 'TRWAGMYFPDXBNJZSQVHLCKE'; + + if (preg_match('/^(\d{8})([A-Z])$/', $value, $m) === 1) { + return $letters[(int) $m[1] % 23] === $m[2]; + } + + if (preg_match('/^([XYZ])(\d{7})([A-Z])$/', $value, $m) === 1) { + $number = (int) (['X' => '0', 'Y' => '1', 'Z' => '2'][$m[1]].$m[2]); + + return $letters[$number % 23] === $m[3]; + } + + return false; + } + + /** + * Determine if a value is a valid Italian fiscal code (the last letter is a check). + */ + public static function codiceFiscale(string $value): bool + { + $value = strtoupper(preg_replace('/\s/', '', $value) ?? ''); + + if (preg_match('/^[A-Z]{6}\d{2}[A-EHLMPRST]\d{2}[A-Z]\d{3}[A-Z]$/', $value) !== 1) { + return false; + } + + $odd = [ + '0' => 1, '1' => 0, '2' => 5, '3' => 7, '4' => 9, '5' => 13, '6' => 15, '7' => 17, '8' => 19, '9' => 21, + 'A' => 1, 'B' => 0, 'C' => 5, 'D' => 7, 'E' => 9, 'F' => 13, 'G' => 15, 'H' => 17, 'I' => 19, 'J' => 21, + 'K' => 2, 'L' => 4, 'M' => 18, 'N' => 20, 'O' => 11, 'P' => 3, 'Q' => 6, 'R' => 8, 'S' => 12, 'T' => 14, + 'U' => 16, 'V' => 10, 'W' => 22, 'X' => 25, 'Y' => 24, 'Z' => 23, + ]; + + $sum = 0; + + for ($i = 0; $i < 15; $i++) { + $char = $value[$i]; + + $sum += $i % 2 === 0 + ? $odd[$char] + : (ctype_digit($char) ? (int) $char : ord($char) - 65); + } + + return chr(65 + $sum % 26) === $value[15]; + } + + /** + * Determine if a value is a valid Belgian national register number (mod-97 check). + */ + public static function belgianNationalNumber(string $value): bool + { + $digits = self::digits($value); + + if (strlen($digits) !== 11) { + return false; + } + + $base = substr($digits, 0, 9); + $check = (int) substr($digits, 9, 2); + + // Births from 2000 on are checked with a leading 2... + return 97 - ((int) $base % 97) === $check + || 97 - ((int) ('2'.$base) % 97) === $check; + } + + /** + * Determine if a value is a valid Swedish personal identity number (a date and a Luhn check). + */ + public static function personnummer(string $value): bool + { + $digits = self::digits($value); + + if (strlen($digits) === 12) { + $digits = substr($digits, 2); + } + + if (strlen($digits) !== 10 || ! self::plausibleDate((int) substr($digits, 2, 2), (int) substr($digits, 4, 2))) { + return false; + } + + return self::luhnSum($digits) % 10 === 0; + } + + /** + * Determine if a value is a valid Norwegian national identity number (two mod-11 checks). + */ + public static function fodselsnummer(string $value): bool + { + $digits = self::digits($value); + + if (strlen($digits) !== 11) { + return false; + } + + $first = self::mod11Check($digits, [3, 7, 6, 1, 8, 9, 4, 5, 2]); + $second = self::mod11Check($digits, [5, 4, 3, 2, 7, 6, 5, 4, 3, 2]); + + return $first === (int) $digits[9] && $second === (int) $digits[10]; + } + + /** + * Determine if a value is a valid Canadian social insurance number (nine digits, Luhn). + */ + public static function sin(string $value): bool + { + $digits = self::digits($value); + + return strlen($digits) === 9 && $digits[0] !== '0' && self::luhnSum($digits) % 10 === 0; + } + + /** + * Determine if a value is a valid Australian tax file number (weighted mod-11). + */ + public static function tfn(string $value): bool + { + $digits = self::digits($value); + + $weights = match (strlen($digits)) { + 9 => [1, 4, 3, 7, 5, 8, 6, 9, 10], + 8 => [10, 7, 8, 4, 6, 3, 5, 1], + default => null, + }; + + if ($weights === null) { + return false; + } + + $sum = 0; + + foreach (str_split($digits) as $i => $digit) { + $sum += (int) $digit * $weights[$i]; + } + + return $sum % 11 === 0; + } + + /** + * Determine if a value is a plausible EU VAT number. + * + * The country prefix selects the check. Countries with a published + * checksum are verified; the rest are accepted on format alone. + */ + public static function vat(string $value): bool + { + $value = strtoupper(preg_replace('/[\s.-]/', '', $value) ?? ''); + + if (preg_match('/^([A-Z]{2})([A-Z0-9]{2,13})$/', $value, $m) !== 1) { + return false; + } + + [, $country, $body] = $m; + + return match ($country) { + 'DE' => preg_match('/^[1-9]\d{8}$/', $body) === 1 && self::vatGermany($body), + 'NL' => preg_match('/^\d{9}B\d{2}$/', $body) === 1 && self::vatNetherlands($body), + 'GB', 'XI' => preg_match('/^\d{9}(\d{3})?$/', $body) === 1 && self::vatBritain($body), + 'IT' => preg_match('/^\d{11}$/', $body) === 1 && self::luhnSum($body) % 10 === 0, + 'FR' => preg_match('/^[A-Z0-9]{2}\d{9}$/', $body) === 1 && self::vatFrance($body), + 'BE' => preg_match('/^[01]\d{9}$/', $body) === 1 && 97 - ((int) substr($body, 0, 8) % 97) === (int) substr($body, 8, 2), + 'ES' => preg_match('/^[A-Z0-9]\d{7}[A-Z0-9]$/', $body) === 1, + 'SE' => preg_match('/^\d{10}01$/', $body) === 1 && self::luhnSum(substr($body, 0, 10)) % 10 === 0, + 'AT' => preg_match('/^U\d{8}$/', $body) === 1, + 'DK' => preg_match('/^\d{8}$/', $body) === 1, + 'FI' => preg_match('/^\d{8}$/', $body) === 1, + 'IE' => preg_match('/^\d[A-Z0-9+*]\d{5}[A-Z]{1,2}$/', $body) === 1, + 'PL' => preg_match('/^\d{10}$/', $body) === 1, + 'PT' => preg_match('/^\d{9}$/', $body) === 1, + 'LU' => preg_match('/^\d{8}$/', $body) === 1, + 'CZ' => preg_match('/^\d{8,10}$/', $body) === 1, + 'HU' => preg_match('/^\d{8}$/', $body) === 1, + 'RO' => preg_match('/^\d{2,10}$/', $body) === 1, + 'SK' => preg_match('/^\d{10}$/', $body) === 1, + 'SI' => preg_match('/^\d{8}$/', $body) === 1, + 'HR' => preg_match('/^\d{11}$/', $body) === 1, + 'BG' => preg_match('/^\d{9,10}$/', $body) === 1, + 'EE', 'LT' => preg_match('/^\d{9}(\d{3})?$/', $body) === 1, + 'LV' => preg_match('/^\d{11}$/', $body) === 1, + 'CY' => preg_match('/^\d{8}[A-Z]$/', $body) === 1, + 'MT' => preg_match('/^\d{8}$/', $body) === 1, + 'EL' => preg_match('/^\d{9}$/', $body) === 1, + default => false, + }; + } + + private static function vatGermany(string $digits): bool + { + $product = 10; + + for ($i = 0; $i < 8; $i++) { + $sum = ((int) $digits[$i] + $product) % 10; + + if ($sum === 0) { + $sum = 10; + } + + $product = ($sum * 2) % 11; + } + + $check = 11 - $product; + + if ($check === 10) { + $check = 0; + } + + return $check === (int) $digits[8]; + } + + private static function vatNetherlands(string $body): bool + { + $sum = 0; + + for ($i = 0; $i < 8; $i++) { + $sum += (int) $body[$i] * (9 - $i); + } + + return $sum % 11 === (int) $body[8]; + } + + private static function vatBritain(string $body): bool + { + $digits = substr($body, 0, 9); + $weights = [8, 7, 6, 5, 4, 3, 2]; + $sum = 0; + + for ($i = 0; $i < 7; $i++) { + $sum += (int) $digits[$i] * $weights[$i]; + } + + $check = (int) substr($digits, 7, 2); + + return ($sum + $check) % 97 === 0 || ($sum + $check + 55) % 97 === 0; + } + + private static function vatFrance(string $body): bool + { + $key = substr($body, 0, 2); + $siren = substr($body, 2); + + if (ctype_digit($key)) { + return (int) $key === (12 + 3 * ((int) $siren % 97)) % 97; + } + + // A key with letters uses a different scheme; accept on format... + return true; + } + + private static function digits(string $value): string + { + return preg_replace('/\D/', '', $value) ?? ''; + } + + private static function luhnSum(string $digits): int + { + $sum = 0; + $double = false; + + for ($i = strlen($digits) - 1; $i >= 0; $i--) { + $digit = (int) $digits[$i]; + + if ($double) { + $digit *= 2; + + if ($digit > 9) { + $digit -= 9; + } + } + + $sum += $digit; + $double = ! $double; + } + + return $sum; + } + + /** + * The remainder of a large numeric string divided by 97, taken piecewise. + */ + private static function mod97(string $number): int + { + $numeric = ''; + + foreach (str_split($number) as $character) { + $numeric .= ctype_alpha($character) ? (string) (ord($character) - 55) : $character; + } + + $remainder = 0; + + foreach (str_split($numeric, 7) as $chunk) { + $remainder = (int) (($remainder).$chunk) % 97; + } + + return $remainder; + } + + /** + * @param array $weights + */ + private static function mod11Check(string $digits, array $weights): int + { + $sum = 0; + + foreach ($weights as $i => $weight) { + $sum += (int) $digits[$i] * $weight; + } + + $check = 11 - ($sum % 11); + + return $check === 11 ? 0 : $check; + } + + private static function plausibleDate(int $month, int $day): bool + { + // Swedish coordination numbers add 60 to the day... + return $month >= 1 && $month <= 12 && (($day >= 1 && $day <= 31) || ($day >= 61 && $day <= 91)); + } +} diff --git a/src/PendingRedaction.php b/src/PendingRedaction.php new file mode 100644 index 0000000..6c6f3b4 --- /dev/null +++ b/src/PendingRedaction.php @@ -0,0 +1,112 @@ +withoutMarkers()->redact($payload); + * Redactor::profile('observability')->inspect($payload)->findings; + */ +class PendingRedaction +{ + use Conditionable; + use Macroable; + + protected ?bool $markers = null; + + protected ?EntityFilter $entities = null; + + public function __construct( + protected Redactor $redactor, + protected ?string $profile = null, + ) {} + + /** + * Use the given profile. + */ + public function profile(?string $profile): static + { + $this->profile = $profile; + + return $this; + } + + /** + * Never write the "_redacted" markers into the payload. + */ + public function withoutMarkers(): static + { + $this->markers = false; + + return $this; + } + + /** + * Write the "_redacted" markers into the payload, whatever the profile says. + */ + public function withMarkers(): static + { + $this->markers = true; + + return $this; + } + + /** + * Act only on the given entities this time. + * + * @param array $entities + */ + public function only(array $entities): static + { + $this->entities = ($this->entities ?? EntityFilter::all())->only($entities); + + return $this; + } + + /** + * Act on every entity but the given ones this time. + * + * @param array $entities + */ + public function except(array $entities): static + { + $this->entities = ($this->entities ?? EntityFilter::all())->except($entities); + + return $this; + } + + /** + * Redact the content and return it. + */ + public function redact(mixed $content): mixed + { + return $this->inspect($content)->value; + } + + /** + * Redact the content and return it with what was found. + */ + public function inspect(mixed $content): RedactionResult + { + return $this->redactor->inspect($content, $this->profile, $this->markers, $this->entities); + } + + /** + * Redact the content without ever throwing, replacing it if redaction fails. + */ + public function redactSafely(mixed $content): mixed + { + try { + return $this->redact($content); + } catch (\Throwable) { + return $this->redactor->redactSafely($content, $this->profile); + } + } +} diff --git a/src/PseudonymizerFactory.php b/src/PseudonymizerFactory.php new file mode 100644 index 0000000..0dee199 --- /dev/null +++ b/src/PseudonymizerFactory.php @@ -0,0 +1,65 @@ +pseudonymization; + + if (($settings['enabled'] ?? true) === false) { + return null; + } + + // The salt is shared across profiles unless one sets its own, since two + // logs must produce the same surrogate for the same user to be joined... + $salt = $settings['salt'] ?? ''; + $salt = is_string($salt) ? $salt : ''; + + try { + $key = $settings['key'] ?? null; + + if (is_string($key) && $key !== '') { + return Pseudonymizer::fromKey($key, $salt); + } + + $applicationKey = Configuration::get('app.key'); + + if (! is_string($applicationKey) || $applicationKey === '') { + InternalLog::warning('Pseudonymization is unavailable: no key configured and app.key is empty', [ + 'profile' => $config->profile, + ]); + + return null; + } + + return Pseudonymizer::derivedFrom($applicationKey, $salt); + } catch (Throwable $e) { + InternalLog::warning('Pseudonymization is unavailable; falling back to plain redaction', [ + 'profile' => $config->profile, + 'reason' => $e->getMessage(), + ]); + + return null; + } + } +} diff --git a/src/Recognition/BatchRecognizer.php b/src/Recognition/BatchRecognizer.php new file mode 100644 index 0000000..e069050 --- /dev/null +++ b/src/Recognition/BatchRecognizer.php @@ -0,0 +1,27 @@ + $texts + * @param array $entities the recogniser's own labels to look for; empty means all + * @return array> spans per input index, offsets relative to that text + * + * @throws \Throwable when the recogniser could not answer + */ + public function recognizeMany(array $texts, string $language, array $entities, float $scoreThreshold): array; +} diff --git a/src/Recognition/CircuitBreaker.php b/src/Recognition/CircuitBreaker.php new file mode 100644 index 0000000..760283b --- /dev/null +++ b/src/Recognition/CircuitBreaker.php @@ -0,0 +1,76 @@ + */ + private static array $state = []; + + /** + * Determine if the recogniser behind the key may be called. + */ + public static function allows(string $key): bool + { + $entry = self::$state[$key] ?? null; + + return $entry === null || $entry['open_until'] <= time(); + } + + /** + * Record a success, closing the breaker. + */ + public static function recordSuccess(string $key): void + { + unset(self::$state[$key]); + } + + /** + * Record a failure and determine if it opened the breaker. + */ + public static function recordFailure(string $key, int $threshold, int $cooldownSeconds): bool + { + $entry = self::$state[$key] ?? ['failures' => 0, 'open_until' => 0]; + $entry['failures']++; + + if ($entry['failures'] >= max(1, $threshold)) { + $entry['open_until'] = time() + max(1, $cooldownSeconds); + $entry['failures'] = 0; + self::$state[$key] = $entry; + + return true; + } + + self::$state[$key] = $entry; + + return false; + } + + /** + * Determine if the breaker is open. + */ + public static function isOpen(string $key): bool + { + return ! self::allows($key); + } + + /** + * Reset all breaker state. + */ + public static function reset(): void + { + self::$state = []; + } +} diff --git a/src/Recognition/RecognizedSpan.php b/src/Recognition/RecognizedSpan.php new file mode 100644 index 0000000..25e5899 --- /dev/null +++ b/src/Recognition/RecognizedSpan.php @@ -0,0 +1,30 @@ + $entities the recogniser's own labels to look for; empty means all + * @return array + * + * @throws \Throwable when the recogniser could not answer + */ + public function recognize(string $text, string $language, array $entities, float $scoreThreshold): array; +} diff --git a/src/Recognition/RecognizerRegistry.php b/src/Recognition/RecognizerRegistry.php new file mode 100644 index 0000000..8cd4351 --- /dev/null +++ b/src/Recognition/RecognizerRegistry.php @@ -0,0 +1,61 @@ + */ + private array $recognizers = []; + + /** + * Create a new recognizer registry instance. + */ + public function __construct() + { + $this->register(new PresidioRecognizer); + } + + /** + * Register a recogniser under its own name. + */ + public function register(Recognizer $recognizer): void + { + $this->recognizers[$recognizer->name()] = $recognizer; + } + + /** + * Determine if a recogniser is registered under the given name. + */ + public function has(string $name): bool + { + return isset($this->recognizers[$name]); + } + + /** + * Get the recogniser registered under the given name, if any. + */ + public function get(string $name): ?Recognizer + { + return $this->recognizers[$name] ?? null; + } + + /** + * Get the registered recogniser names. + * + * @return array + */ + public function names(): array + { + $names = array_keys($this->recognizers); + sort($names); + + return $names; + } +} diff --git a/src/Recognition/Recognizers/PresidioRecognizer.php b/src/Recognition/Recognizers/PresidioRecognizer.php new file mode 100644 index 0000000..fb4ac4e --- /dev/null +++ b/src/Recognition/Recognizers/PresidioRecognizer.php @@ -0,0 +1,206 @@ + $texts + * @param array $entities + * @return array> + * + * @throws RuntimeException + */ + public function recognizeMany(array $texts, string $language, array $entities, float $scoreThreshold): array + { + $results = []; + + foreach ($this->chunk($texts) as $chunk) { + $joined = implode(self::SEPARATOR, $chunk); + $spans = $this->recognize($joined, $language, $entities, $scoreThreshold); + + foreach ($this->segments($chunk) as $index => [$start, $end]) { + $results[$index] = []; + + foreach ($spans as $span) { + if ($span->start >= $start && $span->end <= $end) { + $results[$index][] = new RecognizedSpan($span->entity, $span->start - $start, $span->end - $start, $span->score); + } + } + } + } + + return $results; + } + + /** + * Split the texts into request-sized groups, keeping their indexes. + * + * @param array $texts + * @return array> + */ + private function chunk(array $texts): array + { + $chunks = []; + $current = []; + $length = 0; + $separator = mb_strlen(self::SEPARATOR, 'UTF-8'); + + foreach ($texts as $index => $text) { + $characters = mb_strlen($text, 'UTF-8'); + + if ($current !== [] && $length + $separator + $characters > self::BATCH_CHARACTERS) { + $chunks[] = $current; + $current = []; + $length = 0; + } + + $current[$index] = $text; + $length += ($length === 0 ? 0 : $separator) + $characters; + } + + if ($current !== []) { + $chunks[] = $current; + } + + return $chunks; + } + + /** + * Get the character range each text occupies once joined. + * + * @param array $chunk + * @return array + */ + private function segments(array $chunk): array + { + $segments = []; + $position = 0; + $separator = mb_strlen(self::SEPARATOR, 'UTF-8'); + + foreach ($chunk as $index => $text) { + $characters = mb_strlen($text, 'UTF-8'); + $segments[$index] = [$position, $position + $characters]; + $position += $characters + $separator; + } + + return $segments; + } + + /** + * Recognise entities in the given text using the Presidio analyzer. + * + * @param array $entities + * @return array + * + * @throws RuntimeException + */ + public function recognize(string $text, string $language, array $entities, float $scoreThreshold): array + { + $payload = [ + 'text' => $text, + 'language' => $language, + 'score_threshold' => $scoreThreshold, + ]; + + if ($entities !== []) { + $payload['entities'] = array_values($entities); + } + + $response = Http::timeout($this->timeout) + ->connectTimeout(min(1.0, $this->timeout)) + ->acceptJson() + ->post($this->url, $payload); + + if (! $response->successful()) { + throw new RuntimeException(sprintf('Presidio returned %d.', $response->status())); + } + + $decoded = $response->json(); + + if (! is_array($decoded)) { + throw new RuntimeException('Presidio returned a non-list body.'); + } + + $spans = []; + + foreach ($decoded as $item) { + if (! is_array($item)) { + continue; + } + + $entity = $item['entity_type'] ?? null; + $start = $item['start'] ?? null; + $end = $item['end'] ?? null; + $score = $item['score'] ?? null; + + if (! is_string($entity) || ! is_int($start) || ! is_int($end) || ! is_numeric($score)) { + continue; + } + + $spans[] = new RecognizedSpan($entity, $start, $end, (float) $score); + } + + return $spans; + } +} diff --git a/src/RedactionContext.php b/src/RedactionContext.php index 4e4face..49ced11 100644 --- a/src/RedactionContext.php +++ b/src/RedactionContext.php @@ -4,41 +4,397 @@ namespace Kirschbaum\Redactor; +use Kirschbaum\Redactor\Detection\Confidence; +use Kirschbaum\Redactor\Detection\Detection; +use Kirschbaum\Redactor\Detection\DetectionSet; +use Kirschbaum\Redactor\Detection\EntityFilter; +use Kirschbaum\Redactor\Findings\MatchFinding; +use Kirschbaum\Redactor\Operators\OperatorContext; +use Kirschbaum\Redactor\Operators\OperatorRegistry; +use Kirschbaum\Redactor\Operators\OperatorSpec; +use Kirschbaum\Redactor\Recognition\RecognizedSpan; +use Kirschbaum\Redactor\Recognition\RecognizerRegistry; +use Kirschbaum\Redactor\Support\InternalLog; +use Kirschbaum\Redactor\Support\Pseudonymizer; +use Kirschbaum\Redactor\Support\SecretRegistry; + class RedactionContext { - /** @var array */ + /** @var array */ private array $redactedKeys = []; + /** @var array */ + private array $findings = []; + + /** + * The objects currently on the recursion stack, used to break reference cycles. + * + * Keyed by spl_object_id rather than SplObjectStorage, whose contains() / + * attach() / detach() trio is deprecated in PHP 8.5: a deprecation raised + * inside a log tap becomes a log record, which is redacted, which raises it again. + * + * @var array + */ + private array $activeObjects = []; + + private int $depth = 0; + /** @var array */ private array $entropyCache = []; + /** + * The detections reported for the value currently being processed, not yet acted on. + * + * @var array + */ + private array $pending = []; + public bool $wasRedacted = false; + private ?Pseudonymizer $pseudonymizer = null; + + private bool $pseudonymizerResolved = false; + + private ?SecretRegistry $secrets = null; + public function __construct( - public readonly RedactorConfig $config + public readonly RedactorConfig $config, + public readonly OperatorRegistry $operators = new OperatorRegistry, + /** Secrets registered at runtime, merged with the profile's own. */ + private readonly ?SecretRegistry $runtimeSecrets = null, + private readonly ?RecognizerRegistry $recognizerRegistry = null, + /** Which entities this redaction acts on; null for all of them. */ + private readonly ?EntityFilter $entityFilter = null, ) {} /** - * Add a key to the list of redacted keys. + * Determine if this redaction acts on detections of the given entity. */ - public function addRedactedKey(string $key): void + public function wants(string $entity): bool { - $this->redactedKeys[] = $key; - $this->wasRedacted = true; + return ! $this->entityFilter instanceof EntityFilter || $this->entityFilter->allows($entity); + } + + private ?RecognizerRegistry $defaultRecognizers = null; + + /** + * Spans a priming pass already recognised, keyed by the exact text. + * + * @var array> + */ + private array $recognized = []; + + /** + * Remember what a recogniser found in each text so the walk need not ask again. + * + * @param array> $spansByText + */ + public function primeRecognition(array $spansByText): void + { + $this->recognized = $spansByText + $this->recognized; + } + + /** + * Get the spans already recognised in the exact text, or null if it was never primed. + * + * @return array|null + */ + public function primedRecognition(string $text): ?array + { + return $this->recognized[$text] ?? null; + } + + /** + * Get the recognizer registry. + */ + public function recognizers(): RecognizerRegistry + { + return $this->recognizerRegistry ?? ($this->defaultRecognizers ??= new RecognizerRegistry); + } + + /** + * Get every known secret in play, the profile's plus any registered at runtime. + */ + public function secrets(): SecretRegistry + { + return $this->secrets ??= $this->runtimeSecrets instanceof SecretRegistry + ? $this->config->knownSecrets->merge($this->runtimeSecrets) + : $this->config->knownSecrets; + } + + /** + * Enter one level of nesting, returning false when the max depth would be exceeded. + */ + public function enterDepth(): bool + { + if ($this->depth >= $this->config->maxDepth) { + return false; + } + + $this->depth++; + + return true; + } + + /** + * Leave one level of nesting. + */ + public function leaveDepth(): void + { + if ($this->depth > 0) { + $this->depth--; + } + } + + /** + * Mark an object as being processed, returning false if it is already on the stack. + */ + public function enterObject(object $object): bool + { + $id = spl_object_id($object); + + if (isset($this->activeObjects[$id])) { + return false; + } + + $this->activeObjects[$id] = true; + + return true; + } + + /** + * Mark an object as no longer being processed. + */ + public function leaveObject(object $object): void + { + unset($this->activeObjects[spl_object_id($object)]); } /** * Get all redacted keys. * - * @return array + * @return array */ public function getRedactedKeys(): array { - return array_unique($this->redactedKeys); + return array_values(array_unique($this->redactedKeys)); + } + + /** + * Apply the configured operator to a detection. + * + * The single place a detection turns into a decision, so every strategy + * gets the same precedence rules and the same never-throw behaviour. + */ + public function operate(Detection $detection, ?OperatorSpec $atLocation = null): string + { + $spec = $this->config->policy->operatorFor($detection, $atLocation); + + // Plain redaction is the common case and needs no operator context or registry lookup... + if ($spec->name === OperatorRegistry::REDACT && $spec->options === []) { + return $this->config->replacement; + } + + if (! $this->operators->has($spec->name)) { + InternalLog::warning('Unknown redaction operator; falling back to the replacement string', [ + 'operator' => $spec->name, + 'profile' => $this->config->profile, + 'rule' => $detection->rule, + ]); + + return $this->config->replacement; + } + + return $this->operators->get($spec->name)->apply( + $detection, + new OperatorContext($this->config->replacement, $spec->options, fn (): ?\Kirschbaum\Redactor\Support\Pseudonymizer => $this->pseudonymizer()), + ); + } + + /** + * Get the operator the policy chooses for a detection, without applying it. + * + * For the whole-value sites, a blocked key or a path rule, where remove + * and nullify change the record rather than the text and have to be + * acted on by the walk itself. + */ + public function operatorSpecFor(Detection $detection, ?OperatorSpec $atLocation = null): OperatorSpec + { + return $this->config->policy->operatorFor($detection, $atLocation); + } + + /** + * Determine if the profile's allowlist says the value is never sensitive. + */ + public function isAllowed(string $value): bool + { + return $this->config->allowlist->allows($value); + } + + /** + * Hold a detection until every detector has had its turn on the value. + */ + public function collect(Detection $detection): void + { + $this->pending[] = $detection; + } + + /** + * Determine if any detections are pending. + */ + public function hasPendingDetections(): bool + { + return $this->pending !== []; + } + + /** + * Drop the pending detections because a later strategy settled the value some other way. + */ + public function discardPendingDetections(): void + { + $this->pending = []; + } + + /** + * Act on everything collected for a value, in one pass over it. + * + * The confidence floor and overlap resolution happen here, once, for every + * detector alike. Offsets are trusted because every detector saw this + * exact subject and nothing has rewritten it in between. + */ + public function resolvePendingDetections(string $subject, string $key): string + { + $pending = $this->entityFilter instanceof EntityFilter + ? array_values(array_filter($this->pending, fn (Detection $d): bool => $d->failClosed || $this->wants($d->entity))) + : $this->pending; + + $kept = DetectionSet::resolve($pending, $this->config->minConfidence); + $this->pending = []; + + if ($kept === []) { + return $subject; + } + + $out = ''; + $cursor = 0; + $changed = false; + + foreach ($kept as $detection) { + if ($detection->offset < $cursor) { + // Cannot happen after resolve(), but a bug here would splice + // garbage into a log line, so skipping is the safe failure... + continue; + } + + if (! $detection->failClosed && $this->isAllowed($detection->value)) { + continue; + } + + $replacement = $detection->failClosed + ? $this->config->replacement + : $this->operate($detection); + + if ($replacement === $detection->value) { + // A preserving operator: detected and reported, but deliberately left alone... + $this->recordDetection($detection, redacted: false); + + continue; + } + + $out .= substr($subject, $cursor, $detection->offset - $cursor).$replacement; + $cursor = $detection->end(); + $changed = true; + + $this->recordDetection($detection); + } + + return $changed ? $out.substr($subject, $cursor) : $subject; + } + + /** + * Get the pseudonymizer for this profile, or null when none is configured. + * + * Resolved once and cached, since a misconfigured key must not raise on + * every value in a payload. + */ + public function pseudonymizer(): ?Pseudonymizer + { + if ($this->pseudonymizerResolved) { + return $this->pseudonymizer; + } + + $this->pseudonymizerResolved = true; + $this->pseudonymizer = PseudonymizerFactory::forProfile($this->config); + + return $this->pseudonymizer; + } + + /** + * Record that a rule redacted something under the given key. + * + * An empty key, a bare string passed straight to redact(), sets only the + * redaction flag. + */ + public function recordRedaction( + string $key, + ?string $rule = null, + int $offset = 0, + int $length = 0, + string $matched = '', + ?string $entity = null, + ?Confidence $confidence = null, + bool $redacted = true, + ): void { + if ($redacted) { + $this->wasRedacted = true; + + if ($key !== '') { + $this->redactedKeys[] = $key; + } + } + + if ($rule !== null) { + $this->findings[] = new MatchFinding( + rule: $rule, + key: $key, + offset: $offset, + length: $length, + matched: $matched, + entity: $entity, + confidence: $confidence, + ); + } + } + + /** + * Record a detection, carrying its entity and score through to the report. + */ + public function recordDetection(Detection $detection, bool $redacted = true): void + { + $this->recordRedaction( + key: $detection->key, + rule: $detection->rule, + offset: $detection->offset, + length: $detection->length(), + matched: $detection->value, + entity: $detection->entity, + confidence: $detection->confidence, + redacted: $redacted, + ); + } + + /** + * Get every match recorded during this redaction, in the order found. + * + * @return array + */ + public function getFindings(): array + { + return $this->findings; } /** - * Mark that redaction occurred. + * Mark that a redaction occurred. */ public function markRedacted(): void { @@ -46,7 +402,7 @@ public function markRedacted(): void } /** - * Check if any redaction occurred. + * Determine if any redaction occurred. */ public function hasRedactions(): bool { @@ -54,7 +410,7 @@ public function hasRedactions(): bool } /** - * Get cached entropy for a string. + * Get the cached entropy for a string. */ public function getCachedEntropy(string $string): ?float { @@ -62,7 +418,7 @@ public function getCachedEntropy(string $string): ?float } /** - * Cache entropy calculation for a string. + * Cache the entropy of a string. */ public function cacheEntropy(string $string, float $entropy): void { diff --git a/src/RedactionResult.php b/src/RedactionResult.php new file mode 100644 index 0000000..c3ee8b5 --- /dev/null +++ b/src/RedactionResult.php @@ -0,0 +1,57 @@ + + */ +final readonly class RedactionResult implements Arrayable, JsonSerializable +{ + /** + * @param array $redactedKeys + * @param array $findings + */ + public function __construct( + public mixed $value, + public bool $wasRedacted, + public array $redactedKeys = [], + public array $findings = [], + ) {} + + /** + * Get the result as an array. + * + * @return array + */ + public function toArray(): array + { + return [ + 'value' => $this->value, + 'was_redacted' => $this->wasRedacted, + 'redacted_keys' => $this->redactedKeys, + 'findings' => array_map(fn (MatchFinding $finding): array => $finding->toArray(), $this->findings), + ]; + } + + /** + * Convert the result into something JSON serializable. + * + * @return array + */ + public function jsonSerialize(): array + { + return $this->toArray(); + } +} diff --git a/src/Redactor.php b/src/Redactor.php index 664991e..78cf77a 100644 --- a/src/Redactor.php +++ b/src/Redactor.php @@ -4,109 +4,453 @@ namespace Kirschbaum\Redactor; -use Illuminate\Support\Facades\Config; -use Illuminate\Support\Facades\Log; -use Kirschbaum\Redactor\Strategies\LargeObjectStrategy; -use Kirschbaum\Redactor\Strategies\RedactionStrategyInterface; -use Kirschbaum\Redactor\Strategies\ShannonEntropyStrategy; +use Illuminate\Container\Container; +use Illuminate\Support\Traits\Conditionable; +use Illuminate\Support\Traits\Macroable; +use Kirschbaum\Redactor\Detection\Confidence; +use Kirschbaum\Redactor\Detection\Detection; +use Kirschbaum\Redactor\Detection\EntityFilter; +use Kirschbaum\Redactor\Events\RedactionPerformed; +use Kirschbaum\Redactor\Operators\Operator; +use Kirschbaum\Redactor\Operators\OperatorRegistry; +use Kirschbaum\Redactor\Path\PathCursor; +use Kirschbaum\Redactor\Path\PathMatch; +use Kirschbaum\Redactor\Recognition\Recognizer; +use Kirschbaum\Redactor\Recognition\RecognizerRegistry; +use Kirschbaum\Redactor\Strategies\Contracts\ChainableStrategy; +use Kirschbaum\Redactor\Strategies\Contracts\ConditionalStrategy; +use Kirschbaum\Redactor\Strategies\Contracts\DetectingStrategy; +use Kirschbaum\Redactor\Strategies\Contracts\PreservingStrategy; +use Kirschbaum\Redactor\Strategies\Contracts\PrimingStrategy; +use Kirschbaum\Redactor\Strategies\Contracts\Strategy; +use Kirschbaum\Redactor\Strategies\RegexPatternsStrategy; +use Kirschbaum\Redactor\Strategies\StrategyOutcome; +use Kirschbaum\Redactor\Support\Configuration; +use Kirschbaum\Redactor\Support\InternalLog; +use Kirschbaum\Redactor\Support\SecretRegistry; +use Kirschbaum\Redactor\Tokenization\Detokenizer; class Redactor { - /** @var array> */ + use Conditionable; + use Macroable; + + /** @var array> */ private array $profileStrategies = []; - /** @var array */ + /** @var array */ private array $customStrategies = []; + private bool $customStrategiesLoaded = false; + + private OperatorRegistry $operators; + + private SecretRegistry $secrets; + + private RecognizerRegistry $recognizers; + public function __construct() { - $this->loadCustomStrategies(); + $this->operators = new OperatorRegistry; + $this->secrets = new SecretRegistry; + $this->recognizers = new RecognizerRegistry; + } + + /** + * Register a named entity recognizer, selectable from config by name. + */ + public function registerRecognizer(Recognizer $recognizer): void + { + $this->recognizers->register($recognizer); + } + + /** + * Get the recognizer registry. + */ + public function recognizers(): RecognizerRegistry + { + return $this->recognizers; } /** - * Redact sensitive data from content using strategy pattern. + * Exchange every known token in the content for its original value. * - * @param mixed $content The content to redact - * @param string|null $profile The redaction profile to use (defaults to config default) + * Unknown tokens - expired, foreign, invented by a model - are left as they are. + */ + public function detokenize(mixed $content): mixed + { + /** @var Detokenizer $detokenizer */ + $detokenizer = Container::getInstance()->make(Detokenizer::class); + + return $detokenizer->detokenize($content); + } + + /** + * Register a value that must never appear in output, for every profile. + * + * Meant for credentials that only exist at runtime. Values too short to + * match safely are refused, and false is returned. + */ + public function registerSecret(string $value, string $entity = 'known_secret'): bool + { + return $this->secrets->add($value, $entity); + } + + /** + * Register a custom operator, usable from config by name. + */ + public function registerOperator(string $name, Operator $operator): void + { + $this->operators->register($name, $operator); + } + + /** + * Get the operator registry. + */ + public function operators(): OperatorRegistry + { + return $this->operators; + } + + /** + * Begin a redaction with the given profile. + */ + public function profile(?string $profile): PendingRedaction + { + return new PendingRedaction($this, $profile); + } + + /** + * Redact the content and return it. */ public function redact(mixed $content, ?string $profile = null): mixed + { + return $this->inspect($content, $profile)->value; + } + + /** + * Redact the content and return the redaction metadata alongside it. + * + * The metadata is kept out of the payload rather than written into it. + */ + public function inspect(mixed $content, ?string $profile = null, ?bool $mark = null, ?EntityFilter $entities = null): RedactionResult { $config = RedactorConfig::fromConfig($profile); if (! $config->enabled) { - return $content; + return new RedactionResult($content, false); } - $context = new RedactionContext($config); + $context = new RedactionContext($config, $this->operators, $this->secrets, $this->recognizers, $entities); $strategies = $this->getStrategiesForProfile($config); - $redactedContent = $this->redactRecursively($content, '', $context, $strategies); + // A strategy that pays per call rather than per value sees the whole + // payload once before the walk hands it values one at a time... + foreach ($strategies as $strategy) { + if ($strategy instanceof PrimingStrategy) { + $strategy->prime($content, $context); + } + } + + $redactedContent = $this->redactRecursively($content, '', $context, $strategies, false, $config->paths->cursor()); + + $redactedKeys = $context->getRedactedKeys(); + + // An explicit $mark overrides the profile, since a response or export did not ask for markers... + if (is_array($redactedContent) && $context->hasRedactions() && ($mark ?? $config->markRedacted)) { + $redactedContent = $this->markResultArray($redactedContent, $redactedKeys, $config); + } + + $result = new RedactionResult( + value: $redactedContent, + wasRedacted: $context->hasRedactions(), + redactedKeys: $redactedKeys, + findings: $context->getFindings(), + ); + + if ($result->wasRedacted && $this->eventsEnabled()) { + $this->announce($config->profile, $result); + } + + return $result; + } + + /** + * Determine if redaction events should be dispatched. + */ + private function eventsEnabled(): bool + { + return $this->events ??= (bool) Configuration::get('redactor.events', true); + } + + private ?bool $events = null; + + /** + * Dispatch a RedactionPerformed event carrying names and counts only. + * + * A listener that throws inside the logging pipeline would take the log + * line down with it, so any failure is swallowed. + */ + private function announce(string $profile, RedactionResult $result): void + { + $rules = []; + $entities = []; + + foreach ($result->findings as $finding) { + $rules[$finding->rule] = ($rules[$finding->rule] ?? 0) + 1; + $entities[$finding->entity()] = ($entities[$finding->entity()] ?? 0) + 1; + } + + try { + event(new RedactionPerformed($profile, $result->redactedKeys, $rules, $entities, count($result->findings))); + } catch (\Throwable $e) { + InternalLog::warning('A RedactionPerformed listener failed', [ + 'exception_type' => $e::class, + 'exception_message' => $e->getMessage(), + ]); + } + } + + /** + * Write the legacy `_redacted` markers into the payload, where it is safe. + * + * @param array $array + * @param array $redactedKeys + * @return array + */ + private function markResultArray(array $array, array $redactedKeys, RedactorConfig $config): array + { + // A string key would turn a JSON list into an object once encoded... + if (array_is_list($array) && $array !== []) { + return $array; + } + + // Never clobber a key the caller is already using... + if (array_key_exists('_redacted', $array)) { + InternalLog::warning('Payload already contains a "_redacted" key; redaction markers were not added', [ + 'profile' => $config->profile, + ]); + + return $array; + } + + $array['_redacted'] = true; + + if ($config->trackRedactedKeys && $redactedKeys !== [] && ! array_key_exists('_redacted_keys', $array)) { + $array['_redacted_keys'] = $redactedKeys; + } + + return $array; + } + + /** + * Redact the content without ever throwing. + * + * Meant for the logging pipeline, where an exception would take down the + * whole channel, including the error that explains why. On failure the + * content is replaced wholesale rather than passed through, since data + * whose redaction could not be verified is not safe to emit. + */ + public function redactSafely(mixed $content, ?string $profile = null): mixed + { + try { + return $this->redact($content, $profile); + } catch (\Throwable $e) { + InternalLog::warning('Redaction failed; content replaced as a precaution', [ + 'profile' => $profile, + 'exception_type' => $e::class, + 'exception_message' => $e->getMessage(), + ]); + + return $this->failClosed($profile); + } + } + + /** + * Get the value emitted when redaction could not be completed. + */ + protected function failClosed(?string $profile): string + { + $replacement = '[REDACTED]'; + + try { + $replacement = RedactorConfig::fromConfig($profile)->replacement; + } catch (\Throwable) { + // The config is what failed, so fall back to the documented default... + } + + return $replacement.' (redaction failed)'; + } + + /** + * Resolve every configured profile, collecting the problems found. + * + * Run at deploy time so a bad profile fails the deploy rather than the + * first log line that uses it. + * + * @return array profile name => error message + */ + public function validateProfiles(): array + { + $errors = []; + + foreach ($this->profiles() as $profile) { + try { + $config = RedactorConfig::fromConfig($profile); + + $this->buildStrategiesForProfile($config); - // Only add metadata to array results - if (is_array($redactedContent) && $context->hasRedactions() && $config->markRedacted) { - $redactedContent['_redacted'] = true; + $configured = array_values(array_filter($config->strategies, is_string(...))); - if ($config->trackRedactedKeys && ! empty($context->getRedactedKeys())) { - $redactedContent['_redacted_keys'] = $context->getRedactedKeys(); + $conflicts = array_values(array_intersect($config->safeKeys, $config->blockedKeys)); + + if ($conflicts !== []) { + // SafeKeysStrategy runs first, so a key in both lists is silently never redacted... + $errors[$profile] = 'Keys listed in both safe_keys and blocked_keys (safe_keys wins, so these are never redacted): ' + .implode(', ', $conflicts); + + continue; + } + + // Resolved one by one rather than by count, since a conditional strategy + // the profile switched off still resolves but stays out of the chain... + $unresolved = array_values(array_filter( + $configured, + fn (string $name): bool => ! $this->createStrategyInstance($name) instanceof Strategy + )); + + if ($unresolved !== []) { + $errors[$profile] = 'Unresolvable strategies: '.implode(', ', $unresolved); + + continue; + } + + $failedSamples = $this->checkSamples($config); + + if ($failedSamples !== []) { + $errors[$profile] = implode('; ', $failedSamples); + } + } catch (\Throwable $e) { + $errors[$profile] = $e->getMessage(); + } + } + + return $errors; + } + + /** + * Check every rule's samples through the real detection path, describing each that fails. + * + * @return array + */ + private function checkSamples(RedactorConfig $config): array + { + $strategy = new RegexPatternsStrategy; + $context = new RedactionContext($config, $this->operators, $this->secrets, $this->recognizers); + $problems = []; + + foreach ($config->patterns as $rule) { + foreach ($rule->samples as $sample) { + if (! $this->ruleDetectsIn($strategy, $rule->name, $sample, $context)) { + $problems[] = sprintf('rule "%s" does not detect its sample %s', $rule->name, json_encode($sample)); + } + } + + foreach ($rule->counterSamples as $sample) { + if ($this->ruleDetectsIn($strategy, $rule->name, $sample, $context)) { + $problems[] = sprintf('rule "%s" detects its counter-sample %s', $rule->name, json_encode($sample)); + } + } + } + + return $problems; + } + + /** + * Determine if the given rule detects anything in the subject. + */ + private function ruleDetectsIn(RegexPatternsStrategy $strategy, string $rule, string $subject, RedactionContext $context): bool + { + foreach ($strategy->detect($subject, '', $context) as $detection) { + if ($detection->rule === $rule) { + return true; } } - return $redactedContent ?? $content; + return false; } /** - * Get strategies for a specific profile. + * Get the strategy chain for the given profile. * - * @return array + * @return array */ private function getStrategiesForProfile(RedactorConfig $config): array { - $profileName = $config->profile; + // The redactor is a singleton, so the chain is keyed on the profile's build + // id: a rebuilt profile can never be served a stale chain, including one + // that left out a conditional strategy the old profile had switched off... + $cacheKey = $config->profile.'|'.$config->buildId; + + if (! isset($this->profileStrategies[$cacheKey])) { + // Drop chains built for earlier builds of the same profile... + foreach (array_keys($this->profileStrategies) as $key) { + if (str_starts_with($key, $config->profile.'|')) { + unset($this->profileStrategies[$key]); + } + } - if (! isset($this->profileStrategies[$profileName])) { - $this->profileStrategies[$profileName] = $this->buildStrategiesForProfile($config); + $this->profileStrategies[$cacheKey] = $this->buildStrategiesForProfile($config); } - return $this->profileStrategies[$profileName]; + return $this->profileStrategies[$cacheKey]; } /** - * Build strategies for a profile based on configuration. + * Build the strategy chain for the given profile. * - * @return array + * @return array */ private function buildStrategiesForProfile(RedactorConfig $config): array { $strategies = []; $strategyClasses = $config->strategies; - // Build strategy instances based on config ordering (array order = priority) + // Config order is priority order... foreach ($strategyClasses as $strategyClass) { if (! is_string($strategyClass)) { continue; } - $strategy = $this->createStrategyInstance($strategyClass, $config); + $strategy = $this->createStrategyInstance($strategyClass); - if ($strategy !== null) { - $strategies[] = $strategy; + if (! $strategy instanceof Strategy) { + continue; } + + // A strategy with nothing to do for this profile stays out of the chain... + if ($strategy instanceof ConditionalStrategy && ! $strategy->appliesTo($config)) { + continue; + } + + $strategies[] = $strategy; } return $strategies; } /** - * Create a strategy instance by class string. + * Create a strategy instance by custom name or class string. */ - private function createStrategyInstance(string $strategyClass, RedactorConfig $config): ?RedactionStrategyInterface + private function createStrategyInstance(string $strategyClass): ?Strategy { - // Check for custom strategies first (backward compatibility with name => class mapping) + $this->loadCustomStrategies(); + + // Custom strategies registered by name take precedence... if (isset($this->customStrategies[$strategyClass])) { return clone $this->customStrategies[$strategyClass]; } - // Create strategy instance from class string - if (class_exists($strategyClass) && is_subclass_of($strategyClass, RedactionStrategyInterface::class)) { + if (class_exists($strategyClass) && is_subclass_of($strategyClass, Strategy::class)) { return new $strategyClass; } @@ -114,131 +458,328 @@ private function createStrategyInstance(string $strategyClass, RedactorConfig $c } /** - * Load custom strategies from configuration. + * Load the custom strategies from configuration. */ private function loadCustomStrategies(): void { - $customStrategyClasses = Config::get('redactor.custom_strategies', []); + // Loaded lazily, since the singleton is often built before the config is final... + if ($this->customStrategiesLoaded) { + return; + } + + $this->customStrategiesLoaded = true; + + $customStrategyClasses = Configuration::get('redactor.custom_strategies', []); if (! is_array($customStrategyClasses)) { return; } foreach ($customStrategyClasses as $name => $className) { - if (is_string($className) && is_string($name) && class_exists($className) && is_subclass_of($className, RedactionStrategyInterface::class)) { + if (is_string($className) && is_string($name) && class_exists($className) && is_subclass_of($className, Strategy::class)) { $this->customStrategies[$name] = new $className; } } } /** - * Recursively redact data using strategies. + * Recursively redact the given data using the strategy chain. * - * @param array $strategies + * @param array $strategies */ - protected function redactRecursively(mixed $data, string $key, RedactionContext $context, array $strategies): mixed + protected function redactRecursively( + mixed $data, + string $key, + RedactionContext $context, + array $strategies, + bool $alreadyDispatched = false, + ?PathCursor $cursor = null + ): mixed { + if (! is_array($data) && ! is_object($data)) { + return $this->applyStrategiesToValue($data, $key, $context, $strategies); + } + + // Nothing below may recurse without a depth budget, or a self-referencing + // toArray() or a pathologically nested payload would exhaust memory... + if (! $context->enterDepth()) { + return $this->markDepthExceeded($context); + } + + try { + if (is_array($data)) { + /** @var array $arrayData */ + $arrayData = $data; + + return $this->redactArray($arrayData, $context, $strategies, $alreadyDispatched, $cursor); + } + + return $this->redactObject($data, $key, $context, $strategies, $cursor); + } finally { + $context->leaveDepth(); + } + } + + /** + * The marker meaning "drop this key entirely". + */ + protected const REMOVE_MARKER = '__REDACTOR_REMOVE_OBJECT__'; + + /** + * Apply a path rule to whatever it landed on. + * + * Scalars get the full operator range. Containers only support preserve, + * remove and replace, since masking or pseudonymising an array has no + * defensible meaning; anything else collapses the subtree to the + * replacement string. + */ + protected function applyPathRule(mixed $value, string $key, PathMatch $match, RedactionContext $context): mixed { - if (is_array($data)) { - /** @var array $arrayData */ - $arrayData = $data; + $spec = $match->spec; + + if ($spec->name === OperatorRegistry::PRESERVE) { + return $value; + } + + if ($spec->name === OperatorRegistry::REMOVE) { + $context->recordRedaction($key, 'path:'.$match->pattern); + + return self::REMOVE_MARKER; + } - return $this->redactArray($arrayData, $context, $strategies); + if ($spec->name === OperatorRegistry::NULLIFY) { + $context->recordRedaction($key, 'path:'.$match->pattern); + + return null; } - if (is_object($data)) { - return $this->redactObject($data, $key, $context, $strategies); + if (! is_scalar($value)) { + $context->recordRedaction($key, 'path:'.$match->pattern); + + return $context->config->replacement; + } + + $stringValue = (string) $value; + + if ($context->isAllowed($stringValue)) { + return $value; } - // Apply strategies to scalar values - return $this->applyStrategies($data, $key, $context, $strategies); + $detection = new Detection( + entity: $key, + rule: 'path:'.$match->pattern, + offset: 0, + value: $stringValue, + // A path names the location outright, so there is nothing to be uncertain about... + confidence: Confidence::of(Confidence::CERTAIN, sprintf('path "%s" matched', $match->pattern)), + key: $key, + ); + + $context->recordDetection($detection); + + return $context->operate($detection, $spec); + } + + /** + * Replace a subtree that sits deeper than the configured max depth. + */ + protected function markDepthExceeded(RedactionContext $context): string + { + $context->markRedacted(); + + return sprintf( + '%s (Max depth of %d exceeded)', + $context->config->replacement, + $context->config->maxDepth + ); } /** - * Redact sensitive data from an array. + * Redact the given array. * * @param array $array - * @param array $strategies + * @param array $strategies * @return array */ - protected function redactArray(array $array, RedactionContext $context, array $strategies): array - { - // Check for large arrays first (applies to the whole array) - $arrayAsValue = $this->applyStrategies($array, '', $context, $strategies); - if ($arrayAsValue !== $array) { - // Array was redacted by a strategy (e.g., LargeObjectStrategy) - if (is_array($arrayAsValue)) { + protected function redactArray( + array $array, + RedactionContext $context, + array $strategies, + bool $alreadyDispatched = false, + ?PathCursor $cursor = null + ): array { + // Evaluate the array as a whole unless the caller already ran the chain + // over this value with its real key, which would dispatch every nested + // node twice... + $outcome = $alreadyDispatched + ? null + : $this->applyStrategies($array, '', $context, $strategies); + + if ($outcome instanceof StrategyOutcome && $outcome->value !== $array) { + // A strategy replaced the array wholesale... + if (is_array($outcome->value)) { /** @var array $typedArray */ - $typedArray = $arrayAsValue; + $typedArray = $outcome->value; return $typedArray; } - return ['_redacted_array' => $arrayAsValue]; + return ['_redacted_array' => $outcome->value]; } + // Start from the input rather than an empty array: copy-on-write means a + // subtree that redacts to nothing costs a walk and no copy, and returning + // the original lets the caller's identity check short-circuit... /** @var array $result */ - $result = []; + $result = $array; + $changed = false; foreach ($array as $key => $value) { $keyString = (string) $key; - // Apply strategies to the key-value pair - $processedValue = $this->applyStrategies($value, $keyString, $context, $strategies); + // Paths first: a rule naming this exact location outranks anything + // inferred from the key or contents, and settling it here skips the + // strategy chain and the walk below it entirely... + $childCursor = $cursor?->descend($keyString); + $pathMatch = $childCursor?->match(); - // Handle object removal case - if ($processedValue === '__REDACTOR_REMOVE_OBJECT__') { - continue; // Skip adding this key to the result + // A path names the key it lands on, so the key is the entity the filter sees... + if ($pathMatch instanceof PathMatch && $context->wants($keyString)) { + $decided = $this->applyPathRule($value, $keyString, $pathMatch, $context); + + if ($decided === self::REMOVE_MARKER) { + unset($result[$key]); + $changed = true; + + continue; + } + + if ($decided !== $value) { + $result[$key] = $decided; + $changed = true; + } + + continue; } - // If the value wasn't handled by key-based strategies, process recursively - if ($processedValue === $value && (is_array($value) || is_object($value))) { - $processedValue = $this->redactRecursively($value, $keyString, $context, $strategies); + $outcome = $this->applyStrategies($value, $keyString, $context, $strategies); + $processedValue = $outcome instanceof StrategyOutcome ? $outcome->value : $value; + + if ($processedValue === self::REMOVE_MARKER) { + unset($result[$key]); + $changed = true; + + continue; + } - // Handle object removal case after recursive processing - if ($processedValue === '__REDACTOR_REMOVE_OBJECT__') { - continue; // Skip adding this key to the result + // No strategy claimed this container, so walk into it without running + // the chain over it again... + if (! $outcome instanceof StrategyOutcome && (is_array($value) || is_object($value))) { + $processedValue = $this->redactRecursively( + $value, + $keyString, + $context, + $strategies, + alreadyDispatched: true, + cursor: $childCursor, + ); + + if ($processedValue === self::REMOVE_MARKER) { + unset($result[$key]); + $changed = true; + + continue; } } - $result[(string) $key] = $processedValue; + if ($processedValue !== $value) { + $result[$key] = $processedValue; + $changed = true; + } } - return $result; + return $changed ? $result : $array; } /** - * Redact sensitive data from an object. + * Redact the given object. * - * @param array $strategies + * @param array $strategies */ - protected function redactObject(object $object, string $key, RedactionContext $context, array $strategies): mixed + protected function redactObject(object $object, string $key, RedactionContext $context, array $strategies, ?PathCursor $cursor = null): mixed { - // First, check if the object itself should be redacted by strategies - $objectAsValue = $this->applyStrategies($object, $key, $context, $strategies); - if ($objectAsValue !== $object) { - return $objectAsValue; + $outcome = $this->applyStrategies($object, $key, $context, $strategies); + if ($outcome instanceof StrategyOutcome && $outcome->value !== $object) { + return $outcome->value; + } + + // Some objects are values in their own right and taking them apart + // destroys them: a Throwable has no public properties, so its stack + // trace would become an empty array... + if ($this->isOpaque($object)) { + return $object; } - // Try to convert object to array using toArray() method if available + // An object already on the stack would loop; json_encode() catches this + // itself, but the toArray() path below has no such protection... + if (! $context->enterObject($object)) { + $context->markRedacted(); + + return sprintf( + '%s (Circular reference to %s)', + $context->config->replacement, + $object::class + ); + } + + try { + return $this->redactObjectContents($object, $context, $strategies, $cursor); + } finally { + $context->leaveObject($object); + } + } + + /** + * Determine if an object should pass through the walk whole. + * + * Throwables, dates, enums and closures carry no user-supplied fields, and + * every logging formatter already knows how to render them. Key-based + * rules still apply to them, since the strategy chain runs before this check. + */ + protected function isOpaque(object $object): bool + { + return $object instanceof \Throwable + || $object instanceof \DateTimeInterface + || $object instanceof \DateTimeZone + || $object instanceof \UnitEnum + || $object instanceof \Closure; + } + + /** + * Convert the given object to an array and redact it. + * + * @param array $strategies + */ + protected function redactObjectContents(object $object, RedactionContext $context, array $strategies, ?PathCursor $cursor = null): mixed + { if (method_exists($object, 'toArray')) { try { /** @var array $array */ $array = $object->toArray(); - return $this->redactArray($array, $context, $strategies); + return $this->redactArray($array, $context, $strategies, alreadyDispatched: true, cursor: $cursor); } catch (\Throwable) { - // Fall through to other methods + // Fall through to JSON encoding... } } - // Try JSON encoding first to detect circular references and other issues + // JSON encoding also surfaces circular references and unencodable objects... try { $jsonString = json_encode($object, JSON_THROW_ON_ERROR); $array = json_decode($jsonString, true, 512, JSON_THROW_ON_ERROR); if (! is_array($array)) { - Log::warning('Unable to redact object - JSON decode did not return array', [ - 'object_class' => get_class($object), + InternalLog::warning('Unable to redact object - JSON decode did not return array', [ + 'object_class' => $object::class, 'reason' => 'json_decode_not_array', 'decoded_type' => gettype($array), 'behavior' => $context->config->nonRedactableObjectBehavior, @@ -250,13 +791,13 @@ protected function redactObject(object $object, string $key, RedactionContext $c /** @var array $arrayData */ $arrayData = $array; - return $this->redactArray($arrayData, $context, $strategies); + return $this->redactArray($arrayData, $context, $strategies, alreadyDispatched: true, cursor: $cursor); } catch (\Throwable $e) { - Log::warning('Exception while trying to redact object', [ - 'object_class' => get_class($object), + InternalLog::warning('Exception while trying to redact object', [ + 'object_class' => $object::class, 'reason' => 'exception_during_processing', - 'exception_type' => get_class($e), + 'exception_type' => $e::class, 'exception_message' => $e->getMessage(), 'behavior' => $context->config->nonRedactableObjectBehavior, ]); @@ -266,23 +807,69 @@ protected function redactObject(object $object, string $key, RedactionContext $c } /** - * Apply strategies to a value in priority order. + * Apply the strategies to the given value in priority order. * - * @param array $strategies + * @param array $strategies */ - protected function applyStrategies(mixed $value, string $key, RedactionContext $context, array $strategies): mixed + protected function applyStrategies(mixed $value, string $key, RedactionContext $context, array $strategies): ?StrategyOutcome { + $handled = false; + foreach ($strategies as $strategy) { - if ($strategy->shouldHandle($value, $key, $context)) { - return $strategy->handle($value, $key, $context); + if (! $strategy->shouldHandle($value, $key, $context)) { + continue; + } + + // Detecting strategies only report; their reports are acted on together, + // once, before anything that would change the string they were made + // against gets to run... + if (! $strategy instanceof DetectingStrategy && is_string($value) && $context->hasPendingDetections()) { + $value = $context->resolvePendingDetections($value, $key); + } + + $value = $strategy->handle($value, $key, $context); + $handled = true; + + // A preserving strategy declares the value safe: nothing after it runs + // and the walk does not descend, so "this key is safe" means the same + // thing for a scalar and for the array under it... + if ($strategy instanceof PreservingStrategy) { + $context->discardPendingDetections(); + + return new StrategyOutcome($value, preserved: true); + } + + // A strategy that replaces the value wholesale ends the chain, while a + // chainable one only rewrote part of a string, so the remaining + // strategies still need to inspect what is left... + if (! $strategy instanceof ChainableStrategy) { + $context->discardPendingDetections(); + + return new StrategyOutcome($value); } } - return $value; // No strategy handled this value + if (is_string($value) && $context->hasPendingDetections()) { + $value = $context->resolvePendingDetections($value, $key); + } + + return $handled ? new StrategyOutcome($value) : null; + } + + /** + * Apply the strategies to the given value, returning it unchanged if none applied. + * + * @param array $strategies + */ + protected function applyStrategiesToValue(mixed $value, string $key, RedactionContext $context, array $strategies): mixed + { + $outcome = $this->applyStrategies($value, $key, $context, $strategies); + + return $outcome instanceof StrategyOutcome ? $outcome->value : $value; } /** - * Handle objects that cannot be redacted based on configuration. + * Handle an object that cannot be redacted according to the configured behavior. */ protected function handleNonRedactableObject(object $object, RedactionContext $context): mixed { @@ -290,21 +877,25 @@ protected function handleNonRedactableObject(object $object, RedactionContext $c 'remove' => $this->removeObject($context), 'empty_array' => $this->replaceWithEmptyArray($context), 'redact' => $this->replaceWithRedactionText($object, $context), - default => $object, // 'preserve' or any unknown value + default => $object, // "preserve" or an unknown value... }; } /** - * Remove the object entirely (return a special marker that can be filtered out). + * Get the marker that removes the object from its parent entirely. */ protected function removeObject(RedactionContext $context): string { $context->markRedacted(); - return '__REDACTOR_REMOVE_OBJECT__'; + return self::REMOVE_MARKER; } - /** @return array */ + /** + * Replace the object with an empty array. + * + * @return array + */ protected function replaceWithEmptyArray(RedactionContext $context): array { $context->markRedacted(); @@ -313,130 +904,53 @@ protected function replaceWithEmptyArray(RedactionContext $context): array } /** - * Replace with redaction text. + * Replace the object with the redaction text. */ protected function replaceWithRedactionText(object $object, RedactionContext $context): string { $context->markRedacted(); - return sprintf('%s (Non-redactable object %s)', $context->config->replacement, get_class($object)); + return sprintf('%s (Non-redactable object %s)', $context->config->replacement, $object::class); } /** * Register a custom strategy for use in profiles. */ - public function registerCustomStrategy(string $name, RedactionStrategyInterface $strategy): void + public function registerCustomStrategy(string $name, Strategy $strategy): void { + $this->loadCustomStrategies(); + $this->customStrategies[$name] = $strategy; - // Clear cached profile strategies since we've added a new strategy + // Drop the cached chains so the new strategy is picked up... $this->profileStrategies = []; } /** - * Get available redaction profiles. - * - * @return array - */ - public function getAvailableProfiles(): array - { - /** @var array $profiles */ - $profiles = RedactorConfig::getAvailableProfiles(); - - return $profiles; - } - - /** - * Check if a profile exists. - */ - public function profileExists(string $profile): bool - { - return RedactorConfig::profileExists($profile); - } - - /** - * Get all strategies for a specific profile (for testing/debugging). + * Get the names of the configured profiles. * - * @return array + * @return array */ - public function getStrategies(?string $profile = null): array + public function profiles(): array { - $config = RedactorConfig::fromConfig($profile); - - return $this->getStrategiesForProfile($config); + return array_values(array_map(strval(...), RedactorConfig::profiles())); } - // Legacy methods for backward compatibility - /** - * Add a custom strategy to the redactor. - * - * @deprecated Use registerCustomStrategy instead + * Determine if a profile is configured. */ - public function addStrategy(RedactionStrategyInterface $strategy): void + public function hasProfile(string $profile): bool { - // For backward compatibility, add to default profile - $defaultProfile = Config::get('redactor.default_profile', 'default'); - $this->registerCustomStrategy('custom_'.uniqid(), $strategy); + return RedactorConfig::hasProfile($profile); } /** - * Remove a strategy from the redactor. + * Get the strategy chain a profile resolves to. * - * @deprecated Strategy removal should be handled via profile configuration - */ - public function removeStrategy(string $strategyClass): void - { - // Clear cached strategies to force rebuild - $this->profileStrategies = []; - } - - /** - * Calculate Shannon entropy of a string (for testing purposes). - * Delegates to the ShannonEntropyStrategy. + * @return array */ - public function calculateShannonEntropy(string $string): float + public function strategies(?string $profile = null): array { - $config = RedactorConfig::fromConfig(); - $context = new RedactionContext($config); - $strategies = $this->getStrategiesForProfile($config); - - foreach ($strategies as $strategy) { - if ($strategy instanceof ShannonEntropyStrategy) { - $reflection = new \ReflectionClass($strategy); - $method = $reflection->getMethod('calculateShannonEntropy'); - $method->setAccessible(true); - - $result = $method->invoke($strategy, $string, $context); - - return is_float($result) ? $result : 0.0; - } - } - - return 0.0; - } - - /** - * Check if a string matches common patterns (for testing purposes). - * Delegates to the ShannonEntropyStrategy. - */ - public function isCommonPattern(string $string, RedactorConfig $config): bool - { - $context = new RedactionContext($config); - $strategies = $this->getStrategiesForProfile($config); - - foreach ($strategies as $strategy) { - if ($strategy instanceof ShannonEntropyStrategy) { - $reflection = new \ReflectionClass($strategy); - $method = $reflection->getMethod('isCommonPattern'); - $method->setAccessible(true); - - $result = $method->invoke($strategy, $string, $context); - - return is_bool($result) ? $result : false; - } - } - - return false; + return array_values($this->getStrategiesForProfile(RedactorConfig::fromConfig($profile))); } } diff --git a/src/RedactorConfig.php b/src/RedactorConfig.php index 126ff29..184b69c 100644 --- a/src/RedactorConfig.php +++ b/src/RedactorConfig.php @@ -4,17 +4,96 @@ namespace Kirschbaum\Redactor; -use Illuminate\Support\Facades\Config; +use Kirschbaum\Redactor\Config\ConfigValue; +use Kirschbaum\Redactor\Config\ProfileCache; +use Kirschbaum\Redactor\Exceptions\ConfigurationException; +use Kirschbaum\Redactor\Exceptions\ProfileNotFoundException; +use Kirschbaum\Redactor\Operators\OperatorRegistry; +use Kirschbaum\Redactor\Operators\OperatorSpec; +use Kirschbaum\Redactor\Operators\RedactionPolicy; +use Kirschbaum\Redactor\Path\PathTrie; +use Kirschbaum\Redactor\Patterns\PatternRule; +use Kirschbaum\Redactor\Support\AllowList; +use Kirschbaum\Redactor\Support\Configuration; +use Kirschbaum\Redactor\Support\KeyMatcher; +use Kirschbaum\Redactor\Support\SecretRegistry; readonly class RedactorConfig { + /** @var array */ + public const OBJECT_BEHAVIORS = ['preserve', 'remove', 'empty_array', 'redact']; + + /** + * The behaviours for a string longer than max_value_length. + * + * The default, truncate, keeps the head, scans it and notes what was cut; + * redact replaces the whole value. + */ + public const LARGE_STRING_BEHAVIORS = ['truncate', 'redact']; + + /** + * How deep the redactor will walk before it stops and replaces the rest. + * + * Deep enough for any realistic log context, shallow enough that a cyclic + * or pathologically nested payload cannot exhaust memory. + */ + public const DEFAULT_MAX_DEPTH = 32; + + /** + * The compiled safe-key matcher. + * + * Held here rather than looked up per call: KeyMatcher memoises on the + * pattern list, so finding the cached matcher meant imploding every key, + * measured at 0.203us against 0.050us for the match it was avoiding. + */ + public KeyMatcher $safeKeyMatcher; + + /** The compiled blocked-key matcher. See $safeKeyMatcher. */ + public KeyMatcher $blockedKeyMatcher; + + /** + * The pattern rules ordered by min_length, each paired with its declared position. + * + * Lets the regex strategy stop at the first rule too long to match the + * subject. Declared order still decides an equal-score overlap, through + * the priority on each detection. + * + * @var array + */ + public array $patternsByLength; + + /** + * A short digest of everything that decides what this profile detects. + * + * Two scans with the same fingerprint used the same rules, so a baseline + * that records its fingerprint makes a rules change visible rather than + * silently reinterpreting what "accepted" meant. + */ + public string $rulesetFingerprint; + + /** + * A number unique to this built profile, changing on every rebuild. + * + * Anything cached against a profile can key on it and be sure a rebuilt + * profile is never served a stale derivative. + */ + public int $buildId; + + /** + * The values that are never redacted, whichever detector reports them. + * + * Checked after detection rather than instead of it, so the rules stay as + * strong as they were written. + */ + public AllowList $allowlist; + public function __construct( public bool $enabled, - /** @var array */ + /** @var array */ public array $safeKeys, - /** @var array */ + /** @var array */ public array $blockedKeys, - /** @var array */ + /** @var array */ public array $patterns, public string $replacement, public bool $markRedacted, @@ -28,116 +107,404 @@ public function __construct( /** @var array */ public array $strategies, public string $profile, - ) {} + public int $maxDepth = self::DEFAULT_MAX_DEPTH, + /** The score below which detections are not acted on. */ + public float $minConfidence = 0.0, + public RedactionPolicy $policy = new RedactionPolicy, + /** @var array */ + public array $pseudonymization = [], + /** The path rules, compiled once per profile and consulted before any strategy runs. */ + public PathTrie $paths = new PathTrie, + public string $largeStringBehavior = 'truncate', + ?AllowList $allowlist = null, + /** The application's own credentials, redacted wherever they appear verbatim. */ + public SecretRegistry $knownSecrets = new SecretRegistry, + /** @var array The named entity recognition settings. */ + public array $recognition = [], + ) { + $this->safeKeyMatcher = KeyMatcher::for($this->safeKeys); + $this->blockedKeyMatcher = KeyMatcher::for($this->blockedKeys); + $this->allowlist = $allowlist ?? AllowList::none(); + $this->buildId = ProfileCache::nextBuildId(); + $this->rulesetFingerprint = $this->fingerprint($this->patterns, $this->shannonEntropy, $this->minConfidence, $this->safeKeys, $this->blockedKeys); + + $ordered = []; + $position = 0; + + foreach ($this->patterns as $rule) { + $ordered[] = [$rule, $position++]; + } + + usort($ordered, fn (array $a, array $b): int => $a[0]->minLength <=> $b[0]->minLength ?: $a[1] <=> $b[1]); + + $this->patternsByLength = $ordered; + } /** - * Create a RedactorConfig instance from Laravel configuration. + * Create a new RedactorConfig instance from the application's configuration. */ public static function fromConfig(?string $profile = null): self { - $defaultProfile = Config::get('redactor.default_profile', 'default'); - $profile = $profile ?? (is_string($defaultProfile) ? $defaultProfile : 'default'); + $defaultProfile = Configuration::get('redactor.default_profile', 'default'); + $profile ??= is_string($defaultProfile) ? $defaultProfile : 'default'; - $profiles = Config::get('redactor.profiles', []); + $profiles = Configuration::get('redactor.profiles', []); if (! is_array($profiles) || ! isset($profiles[$profile])) { - throw new \InvalidArgumentException("Redaction profile '".$profile."' not found in configuration."); + throw ProfileNotFoundException::named($profile); } $config = $profiles[$profile]; if (! is_array($config)) { - throw new \InvalidArgumentException("Invalid configuration for profile '".$profile."'."); - } - - $safeKeys = $config['safe_keys'] ?? []; - $blockedKeys = $config['blocked_keys'] ?? []; - $patterns = $config['patterns'] ?? []; - $shannonEntropy = $config['shannon_entropy'] ?? []; - $strategies = $config['strategies'] ?? []; - - // Ensure proper array types for constructor - /** @var array $typedShannonEntropy */ - $typedShannonEntropy = is_array($shannonEntropy) ? $shannonEntropy : []; - /** @var array $typedStrategies */ - $typedStrategies = is_array($strategies) ? $strategies : []; - - return new self( - enabled: is_bool($config['enabled'] ?? true) ? $config['enabled'] ?? true : true, - safeKeys: is_array($safeKeys) ? array_map('strtolower', array_filter($safeKeys, 'is_string')) : [], - blockedKeys: is_array($blockedKeys) ? array_map('strtolower', array_filter($blockedKeys, 'is_string')) : [], - patterns: self::validatePatterns(is_array($patterns) ? $patterns : []), - replacement: is_string($config['replacement'] ?? '[REDACTED]') ? $config['replacement'] ?? '[REDACTED]' : '[REDACTED]', - markRedacted: is_bool($config['mark_redacted'] ?? true) ? $config['mark_redacted'] ?? true : true, - trackRedactedKeys: is_bool($config['track_redacted_keys'] ?? false) ? $config['track_redacted_keys'] ?? false : false, - nonRedactableObjectBehavior: is_string($config['non_redactable_object_behavior'] ?? 'preserve') ? $config['non_redactable_object_behavior'] ?? 'preserve' : 'preserve', - maxValueLength: self::validateMaxValueLength($config['max_value_length'] ?? null), - redactLargeObjects: is_bool($config['redact_large_objects'] ?? true) ? $config['redact_large_objects'] ?? true : true, - maxObjectSize: is_int($config['max_object_size'] ?? 100) ? $config['max_object_size'] ?? 100 : 100, - shannonEntropy: $typedShannonEntropy, - strategies: $typedStrategies, + throw new ConfigurationException("Redaction profile [{$profile}] must be an array."); + } + + // Settings folded in from outside the profile must rebuild it when they + // change, or a rotated salt would keep old and new logs joinable and a + // rotated APP_KEY would go unredacted; validation happens once below... + $regions = ConfigValue::stringList($config['regions'] ?? [], "profiles.{$profile}.regions"); + + $shared = [ + Configuration::get('redactor.pseudonymization'), + self::knownSecretSources($config['known_secrets'] ?? []), + $regions === [] ? null : Configuration::get('redactor.regions'), + ]; + + $cached = ProfileCache::get($profile, $config, $shared); + + if ($cached instanceof RedactorConfig) { + return $cached; + } + + $shannonEntropy = ConfigValue::map($config['shannon_entropy'] ?? [], "profiles.{$profile}.shannon_entropy"); + + // The entropy sub-keys are read on every string and env() hands them over as strings... + if (array_key_exists('enabled', $shannonEntropy)) { + $shannonEntropy['enabled'] = ConfigValue::bool($shannonEntropy['enabled'], true, "profiles.{$profile}.shannon_entropy.enabled"); + } + + if (array_key_exists('threshold', $shannonEntropy)) { + $shannonEntropy['threshold'] = ConfigValue::float($shannonEntropy['threshold'], 4.8, "profiles.{$profile}.shannon_entropy.threshold"); + } + + if (array_key_exists('min_length', $shannonEntropy)) { + $shannonEntropy['min_length'] = ConfigValue::positiveInt($shannonEntropy['min_length'], 25, "profiles.{$profile}.shannon_entropy.min_length"); + } + + $built = new self( + enabled: ConfigValue::bool($config['enabled'] ?? true, true, "profiles.{$profile}.enabled"), + safeKeys: array_map(strtolower(...), ConfigValue::stringList($config['safe_keys'] ?? [], "profiles.{$profile}.safe_keys")), + blockedKeys: array_map(strtolower(...), ConfigValue::stringList($config['blocked_keys'] ?? [], "profiles.{$profile}.blocked_keys")), + patterns: self::buildPatternRules( + ConfigValue::map($config['patterns'] ?? [], "profiles.{$profile}.patterns") + self::regionPatterns($regions, $profile), + $profile + ), + replacement: ConfigValue::string($config['replacement'] ?? '[REDACTED]', '[REDACTED]', "profiles.{$profile}.replacement"), + markRedacted: ConfigValue::bool($config['mark_redacted'] ?? true, true, "profiles.{$profile}.mark_redacted"), + trackRedactedKeys: ConfigValue::bool($config['track_redacted_keys'] ?? false, false, "profiles.{$profile}.track_redacted_keys"), + nonRedactableObjectBehavior: ConfigValue::enum( + $config['non_redactable_object_behavior'] ?? 'preserve', + self::OBJECT_BEHAVIORS, + 'preserve', + "profiles.{$profile}.non_redactable_object_behavior" + ), + maxValueLength: ConfigValue::positiveIntOrNull($config['max_value_length'] ?? null, null, "profiles.{$profile}.max_value_length"), + redactLargeObjects: ConfigValue::bool($config['redact_large_objects'] ?? true, true, "profiles.{$profile}.redact_large_objects"), + maxObjectSize: ConfigValue::positiveIntOrNull($config['max_object_size'] ?? 100, 100, "profiles.{$profile}.max_object_size"), + shannonEntropy: $shannonEntropy, + strategies: ConfigValue::map($config['strategies'] ?? [], "profiles.{$profile}.strategies"), profile: $profile, + maxDepth: ConfigValue::positiveInt($config['max_depth'] ?? self::DEFAULT_MAX_DEPTH, self::DEFAULT_MAX_DEPTH, "profiles.{$profile}.max_depth"), + minConfidence: self::confidenceFloor($config['min_confidence'] ?? 0.0, "profiles.{$profile}.min_confidence"), + policy: self::buildPolicy($config['operators'] ?? [], $profile), + pseudonymization: self::pseudonymizationSettings($config['pseudonymization'] ?? [], $profile), + paths: self::buildPaths($config['paths'] ?? [], $profile), + largeStringBehavior: ConfigValue::enum( + $config['large_string_behavior'] ?? 'truncate', + self::LARGE_STRING_BEHAVIORS, + 'truncate', + "profiles.{$profile}.large_string_behavior" + ), + allowlist: AllowList::for(ConfigValue::stringList($config['allowlist'] ?? [], "profiles.{$profile}.allowlist")), + knownSecrets: self::buildKnownSecrets($config['known_secrets'] ?? [], $profile), + recognition: self::recognitionSettings($config['recognition'] ?? [], $profile), ); + + return ProfileCache::put($profile, $config, $built, $shared); + } + + /** + * Get a digest of the settings that decide what the profile detects. + * + * @param array $patterns + * @param array $entropy + * @param array $safeKeys + * @param array $blockedKeys + */ + private function fingerprint(array $patterns, array $entropy, float $minConfidence, array $safeKeys, array $blockedKeys): string + { + $rules = []; + + foreach ($patterns as $name => $rule) { + $rules[$name] = [ + $rule->pattern, $rule->capture, $rule->validator, $rule->entity(), $rule->confidence, + $rule->mode, $rule->keep, $rule->keywords, $rule->minLength, + $rule->operator?->name, $rule->operator?->options, + ]; + } + + $encoded = json_encode([$rules, $entropy, $minConfidence, $safeKeys, $blockedKeys]); + + return substr(hash('sha256', $encoded === false ? serialize($rules) : $encoded), 0, 16); + } + + /** + * Validate the shape of the recognition settings. + * + * @return array + */ + private static function recognitionSettings(mixed $settings, string $profile): array + { + $map = ConfigValue::map($settings, "profiles.{$profile}.recognition"); + + if (array_key_exists('enabled', $map)) { + $map['enabled'] = ConfigValue::bool($map['enabled'], false, "profiles.{$profile}.recognition.enabled"); + } + + foreach (['min_length', 'max_length', 'min_words', 'failure_threshold', 'cooldown'] as $key) { + if (array_key_exists($key, $map)) { + $map[$key] = ConfigValue::positiveInt($map[$key], 1, "profiles.{$profile}.recognition.{$key}"); + } + } + + foreach (['score_threshold', 'timeout'] as $key) { + if (array_key_exists($key, $map)) { + $map[$key] = ConfigValue::float($map[$key], 0.0, "profiles.{$profile}.recognition.{$key}"); + } + } + + return $map; } /** - * Validate regex patterns and remove invalid ones. + * The pattern definitions of the region packs a profile lists. + * + * A profile's own rule of the same name wins, which is what the array + * union at the call site expresses. + * + * @param array $regions + * @return array * - * @param array $patterns - * @return array + * @throws ConfigurationException when a listed region is not defined */ - private static function validatePatterns(array $patterns): array + private static function regionPatterns(array $regions, string $profile): array { - $validPatterns = []; + if ($regions === []) { + return []; + } + + $packs = ConfigValue::map(Configuration::get('redactor.regions', []), 'regions'); + $patterns = []; - foreach ($patterns as $name => $pattern) { - if (! is_string($pattern)) { - continue; + foreach ($regions as $region) { + if (! isset($packs[$region])) { + throw new ConfigurationException(sprintf( + 'Redactor config [profiles.%s.regions] names an unknown region [%s]. Known: %s.', + $profile, + $region, + implode(', ', array_keys($packs)) + )); } - // Test if the regex pattern is valid - if (@preg_match($pattern, '') !== false) { - $validPatterns[(string) $name] = $pattern; + $patterns += ConfigValue::map($packs[$region], "regions.{$region}"); + } + + return $patterns; + } + + /** + * Collect the profile's known secrets from literal values and config keys. + * + * A config key may point at an array, in which case every string leaf + * under it is registered. Non-string leaves and values too short to be + * safe are skipped silently, since a null secret in a local environment + * must not fail the profile. + */ + private static function buildKnownSecrets(mixed $settings, string $profile): SecretRegistry + { + $map = ConfigValue::map($settings, "profiles.{$profile}.known_secrets"); + + $registry = new SecretRegistry; + + foreach (ConfigValue::stringList($map['values'] ?? [], "profiles.{$profile}.known_secrets.values") as $value) { + $registry->add($value); + } + + foreach (ConfigValue::stringList($map['config'] ?? [], "profiles.{$profile}.known_secrets.config") as $key) { + self::registerLeaves($registry, Configuration::get($key)); + } + + return $registry; + } + + /** + * Get the current values behind the profile's known-secret config keys. + * + * Lets the cache tell when one of them changes. + * + * @return array + */ + private static function knownSecretSources(mixed $settings): array + { + if (! is_array($settings) || ! isset($settings['config']) || ! is_array($settings['config'])) { + return []; + } + + $sources = []; + + foreach ($settings['config'] as $key) { + if (is_string($key)) { + $sources[$key] = Configuration::get($key); } } - return $validPatterns; + return $sources; } /** - * Validate max value length configuration. + * Register every string leaf under the value as a known secret. */ - private static function validateMaxValueLength(mixed $value): ?int + private static function registerLeaves(SecretRegistry $registry, mixed $value): void { - if ($value === null) { - return null; + if (is_string($value)) { + $registry->add($value); + + return; } - if (is_numeric($value) && $value > 0) { - return (int) $value; + if (is_array($value)) { + foreach ($value as $leaf) { + self::registerLeaves($registry, $leaf); + } + } + } + + /** + * Merge the global pseudonymization settings with any profile override. + * + * The key is almost always global so surrogates correlate across every + * profile, while a profile may still set its own salt to break + * correlation deliberately, or switch the feature off. + * + * @return array + */ + private static function pseudonymizationSettings(mixed $profileSettings, string $profile): array + { + $global = ConfigValue::map(Configuration::get('redactor.pseudonymization', []), 'pseudonymization'); + $local = ConfigValue::map($profileSettings, "profiles.{$profile}.pseudonymization"); + + return [...$global, ...array_filter($local, fn ($v): bool => $v !== null)]; + } + + /** + * Compile the profile's path rules. + */ + private static function buildPaths(mixed $paths, string $profile): PathTrie + { + $map = ConfigValue::map($paths, "profiles.{$profile}.paths"); + + $rules = []; + + foreach ($map as $pattern => $definition) { + $rules[(string) $pattern] = OperatorSpec::parse($definition, "profiles.{$profile}.paths.{$pattern}"); + } + + return PathTrie::compile($rules); + } + + /** + * Validate the profile's confidence floor. + */ + private static function confidenceFloor(mixed $value, string $path): float + { + $floor = ConfigValue::float($value, 0.0, $path); + + if ($floor < 0.0 || $floor > 1.0) { + throw new ConfigurationException(sprintf( + 'Redactor config [%s] must be between 0 and 1, got %s.', + $path, + (string) $floor + )); + } + + return $floor; + } + + /** + * Build the per-entity operator policy for a profile. + * + * @param mixed $operators + */ + private static function buildPolicy($operators, string $profile): RedactionPolicy + { + $map = ConfigValue::map($operators, "profiles.{$profile}.operators"); + + $specs = []; + + foreach ($map as $entity => $definition) { + $specs[$entity] = OperatorSpec::parse($definition, "profiles.{$profile}.operators.{$entity}"); + } + + return new RedactionPolicy( + $specs, + $specs['default'] ?? new OperatorSpec(OperatorRegistry::REDACT), + ); + } + + /** + * Turn the configured patterns into rules, dropping uncompilable ones. + * + * @param array $patterns + * @return array + */ + private static function buildPatternRules(array $patterns, string $profile): array + { + $rules = []; + + foreach ($patterns as $name => $definition) { + $rule = PatternRule::fromConfig( + (string) $name, + $definition, + "profiles.{$profile}.patterns.{$name}" + ); + + if ($rule instanceof PatternRule) { + $rules[(string) $name] = $rule; + } } - return null; + return $rules; } /** - * Get the list of available profiles. + * Get the names of the configured profiles. * * @return array */ - public static function getAvailableProfiles(): array + public static function profiles(): array { - $profiles = Config::get('redactor.profiles', []); + $profiles = Configuration::get('redactor.profiles', []); return is_array($profiles) ? array_keys($profiles) : []; } /** - * Check if a profile exists. + * Determine if a profile is configured. */ - public static function profileExists(string $profile): bool + public static function hasProfile(string $profile): bool { - $profiles = Config::get('redactor.profiles', []); + $profiles = Configuration::get('redactor.profiles', []); return is_array($profiles) && isset($profiles[$profile]); } diff --git a/src/RedactorServiceProvider.php b/src/RedactorServiceProvider.php index c38b4eb..673bfe0 100644 --- a/src/RedactorServiceProvider.php +++ b/src/RedactorServiceProvider.php @@ -4,29 +4,135 @@ namespace Kirschbaum\Redactor; +use Illuminate\Contracts\Encryption\StringEncrypter; +use Illuminate\Routing\Router; use Illuminate\Support\ServiceProvider; +use Kirschbaum\Redactor\Config\ConfigValue; use Kirschbaum\Redactor\Console\Commands\RedactorScanCommand; +use Kirschbaum\Redactor\Console\Commands\RedactorValidateCommand; +use Kirschbaum\Redactor\Http\Middleware\RedactResponse; +use Kirschbaum\Redactor\Scanner\LineWindowReader; +use Kirschbaum\Redactor\Scanner\Scanner; +use Kirschbaum\Redactor\Tokenization\CacheTokenStore; +use Kirschbaum\Redactor\Tokenization\Detokenizer; +use Kirschbaum\Redactor\Tokenization\LazyTokenStore; +use Kirschbaum\Redactor\Tokenization\TokenizeOperator; +use Kirschbaum\Redactor\Tokenization\TokenStore; class RedactorServiceProvider extends ServiceProvider { + /** + * Register the package's services. + */ public function register(): void { - $this->app->bind(Redactor::class); + // Merged during register() so a provider that reads redactor.* in its own register() sees it... + $this->mergeConfigFrom(__DIR__.'/../config/redactor.php', 'redactor'); + $this->app->singleton(TokenStore::class, fn (): TokenStore => $this->createTokenStore()); + + $this->app->singleton(Detokenizer::class, fn (): Detokenizer => new Detokenizer($this->app->make(TokenStore::class))); + + $this->app->singleton(Redactor::class, fn (): Redactor => $this->createRedactor()); + + $this->app->singleton(Scanner::class, fn (): Scanner => $this->createScanner()); + } + + /** + * Bootstrap the package's services. + */ + public function boot(): void + { + $this->registerMiddleware(); + + if ($this->app->runningInConsole()) { + $this->registerCommands(); + $this->registerPublishing(); + } + } + + /** + * Create the redactor with the operators that need the container. + */ + protected function createRedactor(): Redactor + { + $redactor = new Redactor; + + // The store is resolved on first use, since nothing needs the cache or encrypter until something is tokenised... + $redactor->registerOperator('tokenize', new TokenizeOperator( + new LazyTokenStore(fn (): TokenStore => $this->app->make(TokenStore::class)) + )); + + return $redactor; + } + + /** + * Create the token store from the configured cache store and TTL. + */ + protected function createTokenStore(): TokenStore + { + $store = config('redactor.tokenization.store'); + $ttl = config('redactor.tokenization.ttl'); + + return new CacheTokenStore( + $this->app->make('cache')->store(is_string($store) && $store !== '' ? $store : null), + $this->app->make(StringEncrypter::class), + $ttl === null || $ttl === '' ? null : ConfigValue::positiveInt($ttl, 86_400, 'tokenization.ttl'), + ); + } + + /** + * Create the file scanner from the scan configuration. + */ + protected function createScanner(): Scanner + { + return new Scanner( + $this->app->make(Redactor::class), + ConfigValue::positiveInt(config('redactor.scan.window_lines'), LineWindowReader::DEFAULT_WINDOW_LINES, 'scan.window_lines'), + ConfigValue::positiveIntOrNull(config('redactor.scan.overlap_lines'), LineWindowReader::DEFAULT_OVERLAP_LINES, 'scan.overlap_lines') ?? 0, + null, + ConfigValue::bool(config('redactor.scan.decode'), true, 'scan.decode'), + ); + } + + /** + * Register the "redact" route middleware alias. + */ + protected function registerMiddleware(): void + { + if (! $this->app->bound('router')) { + return; + } + + /** @var Router $router */ + $router = $this->app->make('router'); + + $router->aliasMiddleware('redact', RedactResponse::class); + } + + /** + * Register the package's console commands. + */ + protected function registerCommands(): void + { $this->commands([ RedactorScanCommand::class, + RedactorValidateCommand::class, ]); } - public function boot(): void + /** + * Register the package's publishable resources. + */ + protected function registerPublishing(): void { $this->publishes([ __DIR__.'/../config/redactor.php' => config_path('redactor.php'), ], 'redactor-config'); - $this->mergeConfigFrom( - __DIR__.'/../config/redactor.php', - 'redactor' - ); + $this->publishes([ + __DIR__.'/../stubs/pre-commit' => base_path('.githooks/pre-commit'), + __DIR__.'/../stubs/redactor-scan.yml' => base_path('.github/workflows/redactor-scan.yml'), + ], 'redactor-ci'); } } diff --git a/src/Scanner/Baseline.php b/src/Scanner/Baseline.php new file mode 100644 index 0000000..c9a05ec --- /dev/null +++ b/src/Scanner/Baseline.php @@ -0,0 +1,114 @@ + $fingerprints + */ + private function __construct( + public readonly array $fingerprints, + public readonly ?string $generatedAt = null, + /** The ruleset fingerprint the baseline was generated under. */ + public readonly ?string $ruleset = null, + ) {} + + public static function empty(): self + { + return new self([]); + } + + /** + * @throws JsonException when the file exists but is not readable as a baseline + */ + public static function load(string $path): self + { + if (! is_file($path)) { + return self::empty(); + } + + $contents = @file_get_contents($path); + + if ($contents === false) { + throw new JsonException("Baseline file [{$path}] could not be read."); + } + + /** @var mixed $decoded */ + $decoded = json_decode($contents, true, 512, JSON_THROW_ON_ERROR); + + if (! is_array($decoded) || ! isset($decoded['findings']) || ! is_array($decoded['findings'])) { + throw new JsonException("Baseline file [{$path}] is missing a \"findings\" array."); + } + + $fingerprints = []; + + foreach ($decoded['findings'] as $entry) { + if (is_string($entry)) { + $fingerprints[$entry] = true; + } elseif (is_array($entry) && isset($entry['fingerprint']) && is_string($entry['fingerprint'])) { + $fingerprints[$entry['fingerprint']] = true; + } + } + + $generatedAt = $decoded['generated_at'] ?? null; + $ruleset = $decoded['ruleset'] ?? null; + + return new self( + $fingerprints, + is_string($generatedAt) ? $generatedAt : null, + is_string($ruleset) ? $ruleset : null, + ); + } + + /** + * @param array $findings + */ + public static function write(string $path, array $findings, string $generatedAt, ?string $ruleset = null): bool + { + $entries = []; + + foreach ($findings as $finding) { + // Path and rule are for a human reading the diff; only the fingerprint is matched... + $entries[$finding->fingerprint] = [ + 'fingerprint' => $finding->fingerprint, + 'rule' => $finding->rule, + 'path' => $finding->path, + ]; + } + + ksort($entries); + + $json = json_encode(array_filter([ + 'version' => 1, + 'generated_at' => $generatedAt, + 'ruleset' => $ruleset, + 'findings' => array_values($entries), + ], fn ($v): bool => $v !== null), JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES); + + if ($json === false) { + return false; + } + + return file_put_contents($path, $json."\n") !== false; + } + + public function isEmpty(): bool + { + return $this->fingerprints === []; + } +} diff --git a/src/Scanner/Decoding/Decoder.php b/src/Scanner/Decoding/Decoder.php new file mode 100644 index 0000000..6207808 --- /dev/null +++ b/src/Scanner/Decoding/Decoder.php @@ -0,0 +1,142 @@ + + */ + public static function derive(string $window): array + { + return [ + ...self::jsonEscaped($window), + ...self::urlEncoded($window), + ...self::base64($window), + ]; + } + + /** + * Get the lines with JSON string escapes, unescaped. + * + * `\/` matters most - json_encode() emits it by default, so every credential + * URL in a JSON file carries it - but `\"` and `\uXXXX` are handled the same way. + * + * @return array + */ + private static function jsonEscaped(string $window): array + { + if (! str_contains($window, '\\')) { + return []; + } + + $subjects = []; + $offset = 0; + + foreach (explode("\n", $window) as $line) { + $length = strlen($line); + + if (preg_match('/\\\\(?:[\/"\\\\bfnrt]|u[0-9a-fA-F]{4})/', $line) === 1) { + $decoded = json_decode('"'.str_replace('"', '\\"', $line).'"'); + + // Undo only what json_decode could interpret, so an escaped backslash is not doubled... + if (is_string($decoded) && $decoded !== $line) { + $subjects[] = new DerivedSubject($decoded, $offset, $length, 'json'); + } + } + + $offset += $length + 1; + } + + return $subjects; + } + + /** + * Get the percent-encoded runs, decoded. + * + * @return array + */ + private static function urlEncoded(string $window): array + { + if (! str_contains($window, '%')) { + return []; + } + + if (preg_match_all('/[A-Za-z0-9_.~:\/?#@!$&\'()*+,;=%-]*(?:%[0-9A-Fa-f]{2})+[A-Za-z0-9_.~:\/?#@!$&\'()*+,;=%-]*/', $window, $matches, PREG_OFFSET_CAPTURE) === false) { + return []; + } + + $subjects = []; + + foreach ($matches[0] as [$encoded, $offset]) { + $decoded = rawurldecode($encoded); + + if ($decoded !== $encoded) { + $subjects[] = new DerivedSubject($decoded, $offset, strlen($encoded), 'url'); + } + } + + return $subjects; + } + + /** + * Get the base64 tokens that decode to printable text. + * + * @return array + */ + private static function base64(string $window): array + { + if (preg_match_all('/(?= strlen($bytes) * 0.9; + } +} diff --git a/src/Scanner/Decoding/DerivedSubject.php b/src/Scanner/Decoding/DerivedSubject.php new file mode 100644 index 0000000..5817218 --- /dev/null +++ b/src/Scanner/Decoding/DerivedSubject.php @@ -0,0 +1,23 @@ + $paths Base paths to search (files or directories) - * @param array $excludePatterns Glob-style patterns (e.g., ['*.min.js', 'node_modules/*']) - * @param int $maxSizeBytes Max file size to include (default 10MB) - * @return array Real paths of matched files + * @param array $paths + * @param array $excludePatterns globs matched against the basename and the path relative to each scanned directory + * @return array */ - public static function collect(array $paths, array $excludePatterns = [], int $maxSizeBytes = 10_485_760): array - { + public static function collect( + array $paths, + array $excludePatterns = [], + int $maxSizeBytes = 10_485_760, + bool $skipBinary = true, + bool $respectGitignore = true, + ): array { $files = []; $directoriesToScan = []; - // Separate individual files from directories foreach ($paths as $path) { if (is_file($path)) { - // Handle individual files - if (self::isFileEligible($path, $maxSizeBytes)) { + // An explicitly named file is scanned even if a pattern would exclude it... + if (self::isFileEligible($path, $maxSizeBytes, $skipBinary)) { $realPath = realpath($path); if ($realPath !== false) { $files[] = $realPath; @@ -34,27 +43,41 @@ public static function collect(array $paths, array $excludePatterns = [], int $m } elseif (is_dir($path)) { $directoriesToScan[] = $path; } - // Non-existent paths are silently ignored (command handles warnings) + // Non-existent paths are ignored here; the command warns about them... } - // Process directories with Finder foreach ($directoriesToScan as $directory) { + // Resolve symlinks first: Symfony locates the git root by walking up the given + // path, so a symlinked path makes ignoreVCSIgnored() silently do nothing... + $directory = realpath($directory) ?: $directory; + $finder = (new Finder) ->files() ->ignoreDotFiles(false) ->ignoreVCS(false) ->in($directory); - foreach ($excludePatterns as $pattern) { - $finder->notName($pattern); + if ($respectGitignore) { + $finder->ignoreVCSIgnored(true); + } + + // Prune whole directories during traversal, or 'vendor/*' walks every file under vendor first... + foreach (self::directoryPrefixes($excludePatterns) as $prefix) { + $finder->exclude($prefix); } foreach ($finder as $file) { - if (self::isFileEligible($file->getPathname(), $maxSizeBytes)) { - $realPath = $file->getRealPath(); - if ($realPath !== false) { - $files[] = $realPath; - } + if (self::isExcluded($file, $excludePatterns)) { + continue; + } + + if (! self::isFileEligible($file->getPathname(), $maxSizeBytes, $skipBinary)) { + continue; + } + + $realPath = $file->getRealPath(); + if ($realPath !== false) { + $files[] = $realPath; } } } @@ -63,18 +86,127 @@ public static function collect(array $paths, array $excludePatterns = [], int $m } /** - * Check if a file is eligible for scanning. + * Determine if the file matches any exclude pattern. + * + * Patterns are tested against both the basename and the path relative to + * the scanned directory: Symfony's notName() compares the basename only, + * so 'vendor/*' would never match anything. + * + * @param array $excludePatterns + */ + private static function isExcluded(SplFileInfo $file, array $excludePatterns): bool + { + if ($excludePatterns === []) { + return false; + } + + $basename = $file->getFilename(); + $relativePath = str_replace('\\', '/', $file->getRelativePathname()); + + foreach ($excludePatterns as $pattern) { + if ($pattern === '') { + continue; + } + + if (fnmatch($pattern, $basename) || fnmatch($pattern, $relativePath)) { + return true; + } + } + + return false; + } + + /** + * Determine if the repository-relative path matches any exclude pattern. + * + * The same test isExcluded() applies to walked files, for paths that + * arrive from git rather than from the filesystem. + * + * @param array $excludePatterns + */ + public static function matchesExclude(string $relativePath, array $excludePatterns): bool + { + $relativePath = str_replace('\\', '/', $relativePath); + $basename = basename($relativePath); + + foreach ($excludePatterns as $pattern) { + if ($pattern !== '' && (fnmatch($pattern, $basename) || fnmatch($pattern, $relativePath))) { + return true; + } + } + + return false; + } + + /** + * Get the directory prefixes that can be pruned during traversal. + * + * 'vendor/*' and 'node_modules/**' both mean "skip that directory". + * + * @param array $excludePatterns + * @return array + */ + private static function directoryPrefixes(array $excludePatterns): array + { + $prefixes = []; + + foreach ($excludePatterns as $pattern) { + if (! preg_match('#^([^*?\[\]]+)/\*{1,2}$#', $pattern, $matches)) { + continue; + } + + $prefixes[] = trim($matches[1], '/'); + } + + return array_values(array_unique(array_filter($prefixes))); + } + + /** + * Determine if the file is eligible for scanning. */ - private static function isFileEligible(string $filePath, int $maxSizeBytes): bool + private static function isFileEligible(string $filePath, int $maxSizeBytes, bool $skipBinary = true): bool { if (! is_readable($filePath)) { return false; } - if (filesize($filePath) > $maxSizeBytes) { + $size = @filesize($filePath); + + // filesize() returns false for a file that vanished since the walk; treat that as ineligible... + if ($size === false || $size > $maxSizeBytes) { return false; } - return true; + return ! $skipBinary || ! self::looksBinary($filePath); + } + + /** + * Determine if the file looks like binary content. + * + * Scanning an image or a compiled artefact produces nothing but entropy + * false positives, and reads the whole thing into memory to do it. + */ + private static function looksBinary(string $filePath): bool + { + $sample = @file_get_contents($filePath, false, null, 0, self::BINARY_SNIFF_BYTES); + + if ($sample === false || $sample === '') { + return false; + } + + // A NUL byte is the standard heuristic - git uses the same one... + if (str_contains($sample, "\0")) { + return true; + } + + if (mb_check_encoding($sample, 'UTF-8')) { + return false; + } + + // Not UTF-8, so judge it by bytes: text in a legacy encoding has almost no + // C0 or C1 control bytes, while random binary is a quarter of them... + $control = preg_match_all('/[\x00-\x08\x0E-\x1F\x7F-\x9F]/', $sample); + + return $control > strlen($sample) * 0.05; } } diff --git a/src/Scanner/Git/GitRepository.php b/src/Scanner/Git/GitRepository.php new file mode 100644 index 0000000..aa5e86c --- /dev/null +++ b/src/Scanner/Git/GitRepository.php @@ -0,0 +1,114 @@ +process(['rev-parse', '--is-inside-work-tree']); + + return $process->run() === 0 && trim($process->getOutput()) === 'true'; + } + + /** + * Get the lines added by the changes currently staged for commit. + * + * @param array $pathspec + * @return array + */ + public function staged(array $pathspec = []): array + { + return PatchParser::parse($this->run([ + 'diff', '--cached', '-U0', '--no-color', '--no-ext-diff', '--diff-filter=ACMR', ...$this->spec($pathspec), + ])); + } + + /** + * Get the lines the working tree adds over a ref, such as a branch over main. + * + * @param array $pathspec + * @return array + */ + public function diff(string $ref, array $pathspec = []): array + { + return PatchParser::parse($this->run([ + 'diff', '-U0', '--no-color', '--no-ext-diff', '--diff-filter=ACMR', $ref, ...$this->spec($pathspec), + ])); + } + + /** + * Get the lines added by every commit in a range, newest first, with the commit that added each. + * + * A secret committed and removed two commits later is still in the + * repository's history; this is the mode that finds it. + * + * @param array $pathspec + * @return array + */ + public function history(?string $range = null, array $pathspec = []): array + { + $arguments = ['log', '-p', '-U0', '--no-color', '--no-ext-diff', '--diff-filter=ACMR', '--format=commit %H']; + + if ($range !== null && $range !== '') { + $arguments[] = $range; + } + + return PatchParser::parse($this->run([...$arguments, ...$this->spec($pathspec)])); + } + + /** + * @param array $pathspec + * @return array + */ + private function spec(array $pathspec): array + { + return $pathspec === [] ? [] : ['--', ...$pathspec]; + } + + /** + * @param array $arguments + */ + private function run(array $arguments): string + { + $process = $this->process($arguments); + $process->run(); + + if (! $process->isSuccessful()) { + throw new GitException(sprintf( + 'git %s failed: %s', + $arguments[0], + trim($process->getErrorOutput()) ?: 'exit code '.$process->getExitCode() + )); + } + + return $process->getOutput(); + } + + /** + * @param array $arguments + */ + private function process(array $arguments): Process + { + $process = new Process(['git', ...$arguments], $this->directory); + $process->setTimeout(null); + + return $process; + } +} diff --git a/src/Scanner/Git/Patch.php b/src/Scanner/Git/Patch.php new file mode 100644 index 0000000..211d663 --- /dev/null +++ b/src/Scanner/Git/Patch.php @@ -0,0 +1,43 @@ + $addedLines real line number in the new file => text + */ + public function __construct( + public string $path, + public array $addedLines, + public ?string $commit = null, + ) {} + + /** + * Get the added lines as one text, in order, for scanning. + */ + public function text(): string + { + return implode("\n", array_values($this->addedLines)); + } + + /** + * Get the real file line for the Nth line (1-based) of text(). + */ + public function lineAt(int $textLine): int + { + $keys = array_keys($this->addedLines); + + return $keys[$textLine - 1] ?? $textLine; + } +} diff --git a/src/Scanner/Git/PatchParser.php b/src/Scanner/Git/PatchParser.php new file mode 100644 index 0000000..cc35ccc --- /dev/null +++ b/src/Scanner/Git/PatchParser.php @@ -0,0 +1,126 @@ +` lines between changes. Zero context lines are + * assumed but not required: context and removed lines are simply skipped. + */ +class PatchParser +{ + /** @var array */ + private array $patches = []; + + private ?string $commit = null; + + private ?string $path = null; + + /** @var array */ + private array $added = []; + + private int $line = 0; + + private bool $binary = false; + + private function __construct() {} + + /** + * @return array + */ + public static function parse(string $diff): array + { + $parser = new self; + + foreach (preg_split('/\r?\n/', $diff) ?: [] as $raw) { + $parser->consume($raw); + } + + $parser->flush(); + + return $parser->patches; + } + + private function consume(string $raw): void + { + if (str_starts_with($raw, 'commit ') && preg_match('/^commit ([0-9a-f]{7,40})\b/', $raw, $m) === 1) { + $this->flush(); + $this->commit = $m[1]; + + return; + } + + if (str_starts_with($raw, 'diff --git ')) { + $this->flush(); + + return; + } + + if (str_starts_with($raw, '+++ ')) { + $target = substr($raw, 4); + + // A deleted file has nothing to scan... + $this->path = $target === '/dev/null' ? null : $this->unquote($target); + + return; + } + + if (str_starts_with($raw, 'Binary files ')) { + $this->binary = true; + + return; + } + + if (str_starts_with($raw, '@@ ')) { + // Hunk header: @@ -old[,count] +new[,count] @@... + $this->line = preg_match('/\+(\d+)/', $raw, $m) === 1 ? (int) $m[1] : 1; + + return; + } + + if ($this->path === null) { + return; + } + + if (str_starts_with($raw, '+')) { + $this->added[$this->line] = substr($raw, 1); + $this->line++; + + return; + } + + if (str_starts_with($raw, ' ')) { + // A context line, present when the diff was not made with -U0... + $this->line++; + } + + // '-' lines and '\ No newline at end of file' advance nothing... + } + + private function flush(): void + { + if ($this->path !== null && ! $this->binary && $this->added !== []) { + $this->patches[] = new Patch($this->path, $this->added, $this->commit); + } + + $this->path = null; + $this->added = []; + $this->binary = false; + } + + /** + * Strip the a/ or b/ prefix and undo git's C-style quoting. + */ + private function unquote(string $target): string + { + if (str_starts_with($target, '"') && str_ends_with($target, '"')) { + $target = stripcslashes(substr($target, 1, -1)); + } + + return preg_replace('#^[ab]/#', '', $target) ?? $target; + } +} diff --git a/src/Scanner/JunitReport.php b/src/Scanner/JunitReport.php new file mode 100644 index 0000000..5b88781 --- /dev/null +++ b/src/Scanner/JunitReport.php @@ -0,0 +1,55 @@ + $results + */ + public static function build(array $results): string + { + $failures = 0; + $cases = ''; + + foreach ($results as $result) { + $cases .= sprintf(' '."\n", self::escape($result->path)); + + if ($result->skipped) { + $cases .= sprintf(' '."\n", self::escape($result->error ?? 'skipped')); + } + + foreach ($result->findings as $finding) { + $failures++; + $cases .= sprintf( + ' %s'."\n", + self::escape(sprintf('%s at %s:%d:%d (%s)', $finding->rule, $finding->path, $finding->line, $finding->column, $finding->severity())), + self::escape($finding->rule), + self::escape($finding->excerpt) + ); + } + + $cases .= " \n"; + } + + $count = count($results); + + return ''."\n" + .sprintf(''."\n", $count, $failures) + .sprintf(' '."\n", $count, $failures) + .$cases + ." \n" + ."\n"; + } + + private static function escape(string $value): string + { + return htmlspecialchars($value, ENT_XML1 | ENT_QUOTES, 'UTF-8'); + } +} diff --git a/src/Scanner/LineWindowReader.php b/src/Scanner/LineWindowReader.php new file mode 100644 index 0000000..c39bb22 --- /dev/null +++ b/src/Scanner/LineWindowReader.php @@ -0,0 +1,90 @@ + + */ +class LineWindowReader implements IteratorAggregate +{ + public const DEFAULT_WINDOW_LINES = 512; + + public const DEFAULT_OVERLAP_LINES = 4; + + public function __construct( + private readonly string $path, + private readonly int $windowLines = self::DEFAULT_WINDOW_LINES, + private readonly int $overlapLines = self::DEFAULT_OVERLAP_LINES, + private readonly ?string $content = null, + ) {} + + /** + * Create a reader over in-memory text, such as a git patch. + */ + public static function ofString(string $content, int $windowLines = self::DEFAULT_WINDOW_LINES, int $overlapLines = self::DEFAULT_OVERLAP_LINES): self + { + return new self('php://temp', $windowLines, $overlapLines, $content); + } + + /** + * @return Generator [first line number, window text] + */ + public function getIterator(): Generator + { + $handle = $this->content !== null + ? fopen('php://temp', 'r+b') + : @fopen($this->path, 'rb'); + + if ($handle === false) { + return; + } + + if ($this->content !== null) { + fwrite($handle, $this->content); + rewind($handle); + } + + // Overlap must be smaller than the window, or the reader never advances... + $window = max(1, $this->windowLines); + $overlap = max(0, min($this->overlapLines, $window - 1)); + + try { + $buffer = []; + $startLine = 1; + + while (($line = fgets($handle)) !== false) { + $buffer[] = rtrim($line, "\r\n"); + + if (count($buffer) < $window) { + continue; + } + + yield [$startLine, implode("\n", $buffer)]; + + // Carry the tail forward so the next window sees a match that began in this one... + $carried = $overlap > 0 ? array_slice($buffer, -$overlap) : []; + $startLine += count($buffer) - count($carried); + $buffer = $carried; + } + + // The final partial window, unless it holds nothing but the overlap already emitted... + if ($buffer !== [] && ($startLine === 1 || count($buffer) > $overlap)) { + yield [$startLine, implode("\n", $buffer)]; + } + } finally { + fclose($handle); + } + } +} diff --git a/src/Scanner/SarifReport.php b/src/Scanner/SarifReport.php new file mode 100644 index 0000000..0e5d6aa --- /dev/null +++ b/src/Scanner/SarifReport.php @@ -0,0 +1,85 @@ + $findings + * @return array + */ + public static function build(array $findings, string $version = '1.0.0', ?string $ruleset = null): array + { + $rules = []; + $results = []; + + foreach ($findings as $finding) { + $rules[$finding->rule] ??= [ + 'id' => $finding->rule, + 'name' => $finding->rule, + 'shortDescription' => ['text' => sprintf('Potential secret detected by rule "%s"', $finding->rule)], + 'defaultConfiguration' => ['level' => 'error'], + ]; + + // Map the score onto SARIF's levels so a low-confidence hit is a note, not a merge blocker... + $level = match ($finding->severity()) { + 'high' => 'error', + 'medium' => 'warning', + default => 'note', + }; + + $results[] = [ + 'ruleId' => $finding->rule, + 'level' => $level, + 'message' => ['text' => sprintf( + 'Sensitive content matched rule "%s"%s.', + $finding->rule, + $finding->confidence === null ? '' : sprintf(' (confidence %.2f)', $finding->confidence) + )], + 'partialFingerprints' => ['redactorFingerprint/v1' => $finding->fingerprint], + 'properties' => array_filter([ + 'entity' => $finding->entity, + 'confidence' => $finding->confidence, + 'signals' => $finding->signals, + ], fn (string|float|array|null $v): bool => ! in_array($v, [null, '', []], true)), + 'locations' => [[ + 'physicalLocation' => [ + 'artifactLocation' => ['uri' => $finding->path], + 'region' => [ + 'startLine' => max(1, $finding->line), + 'startColumn' => max(1, $finding->column), + // The snippet is redacted output, so the file can be uploaded without the secret... + 'snippet' => ['text' => $finding->excerpt], + ], + ], + ]], + ]; + } + + return [ + '$schema' => self::SCHEMA, + 'version' => '2.1.0', + 'runs' => [[ + 'tool' => [ + 'driver' => [ + 'name' => 'Redactor', + 'informationUri' => 'https://github.com/kirschbaum-development/redactor', + 'version' => $version, + 'rules' => array_values($rules), + // Record which rules produced these results so two runs can be compared... + 'properties' => array_filter(['rulesetFingerprint' => $ruleset]), + ], + ], + 'results' => $results, + ]], + ]; + } +} diff --git a/src/Scanner/ScanFinding.php b/src/Scanner/ScanFinding.php new file mode 100644 index 0000000..4db420b --- /dev/null +++ b/src/Scanner/ScanFinding.php @@ -0,0 +1,152 @@ + + */ +final readonly class ScanFinding implements Arrayable, JsonSerializable +{ + public function __construct( + public string $path, + public string $rule, + public int $line, + public int $column, + public string $excerpt, + public string $profile, + public string $fingerprint, + public string $entity = '', + /** 0.0-1.0, or null where the rule is structural rather than inferred. */ + public ?float $confidence = null, + /** @var array */ + public array $signals = [], + /** Set only when verification ran; never carries the secret itself. */ + public ?VerificationResult $verification = null, + /** The commit that added the line, when scanning git history. */ + public ?string $commit = null, + /** base64, url or json when the secret was found inside an encoded span. */ + public ?string $encoding = null, + ) {} + + public function withVerification(VerificationResult $result): self + { + return new self( + path: $this->path, + rule: $this->rule, + line: $this->line, + column: $this->column, + excerpt: $this->excerpt, + profile: $this->profile, + fingerprint: $this->fingerprint, + entity: $this->entity, + confidence: $this->confidence, + signals: $this->signals, + verification: $result, + commit: $this->commit, + encoding: $this->encoding, + ); + } + + /** + * Relocate the finding to its real line in the file and the commit that added it. + */ + public function at(int $line, ?string $commit): self + { + return new self( + path: $this->path, + rule: $this->rule, + line: $line, + column: $this->column, + excerpt: $this->excerpt, + profile: $this->profile, + fingerprint: $this->fingerprint, + entity: $this->entity, + confidence: $this->confidence, + signals: $this->signals, + verification: $this->verification, + commit: $commit, + encoding: $this->encoding, + ); + } + + /** + * Get the finding's location as a human reads it. + */ + public function location(): string + { + $where = sprintf('%s:%d:%d', $this->path, $this->line, $this->column); + + return $this->commit === null ? $where : substr($this->commit, 0, 8).':'.$where; + } + + /** + * Get a severity a human can sort by. + */ + public function severity(): string + { + // A confirmed-live credential outranks anything confidence can say... + if ($this->verification instanceof VerificationResult && $this->verification->status->isActive()) { + return 'critical'; + } + + return match (true) { + $this->confidence === null => 'high', + $this->confidence >= 0.9 => 'high', + $this->confidence >= 0.6 => 'medium', + $this->confidence >= 0.3 => 'low', + default => 'very-low', + }; + } + + /** + * @return array + */ + public function toArray(): array + { + return [ + 'rule' => $this->rule, + 'entity' => $this->entity, + 'line' => $this->line, + 'column' => $this->column, + 'excerpt' => $this->excerpt, + 'confidence' => $this->confidence, + 'severity' => $this->severity(), + // Why the score is what it is, so a threshold can be chosen on evidence... + 'signals' => $this->signals, + 'verification' => $this->verification?->toArray(), + 'commit' => $this->commit, + 'encoding' => $this->encoding, + 'profile' => $this->profile, + 'fingerprint' => $this->fingerprint, + ]; + } + + /** + * @return array + */ + public function jsonSerialize(): array + { + return $this->toArray(); + } + + /** + * Get a stable identity for the finding. + * + * Derived from the rule, the file and the secret itself - never the line + * number, so a finding accepted into a baseline stays accepted when the + * code above it moves. The secret is hashed, never stored. + */ + public static function fingerprint(string $rule, string $path, string $matched): string + { + return substr(hash('sha256', $rule.'|'.$path.'|'.$matched), 0, 32); + } +} diff --git a/src/Scanner/ScanResult.php b/src/Scanner/ScanResult.php index 771ef73..dac6b5a 100644 --- a/src/Scanner/ScanResult.php +++ b/src/Scanner/ScanResult.php @@ -7,7 +7,7 @@ class ScanResult { /** - * @param array> $findings + * @param array $findings */ public function __construct( public readonly string $path, @@ -21,4 +21,27 @@ public function hasFindings(): bool { return count($this->findings) > 0; } + + /** + * Get the same result with any baseline-accepted findings removed. + * + * @param array $acceptedFingerprints + */ + public function withoutBaseline(array $acceptedFingerprints): self + { + if ($acceptedFingerprints === [] || ! $this->hasFindings()) { + return $this; + } + + return new self( + path: $this->path, + findings: array_values(array_filter( + $this->findings, + fn (ScanFinding $finding): bool => ! isset($acceptedFingerprints[$finding->fingerprint]) + )), + profile: $this->profile, + skipped: $this->skipped, + error: $this->error, + ); + } } diff --git a/src/Scanner/Scanner.php b/src/Scanner/Scanner.php index 5dbc2f8..9895d8c 100644 --- a/src/Scanner/Scanner.php +++ b/src/Scanner/Scanner.php @@ -4,19 +4,56 @@ namespace Kirschbaum\Redactor\Scanner; +use Kirschbaum\Redactor\Findings\MatchFinding; use Kirschbaum\Redactor\Redactor; +use Kirschbaum\Redactor\Scanner\Decoding\Decoder; +use Kirschbaum\Redactor\Scanner\Decoding\DerivedSubject; +use Kirschbaum\Redactor\Scanner\Git\Patch; +use Kirschbaum\Redactor\Verification\SecretVerifier; +use Kirschbaum\Redactor\Verification\VerificationResult; class Scanner { + /** + * How much of a line to show in a finding's excerpt. + */ + private const int EXCERPT_LIMIT = 200; + + /** + * A marker on the same line that suppresses the finding. + * + * $key = 'sk_test_4eC39HqLyjWDarjtT1zdp7dc'; // redactor:allow + * + * For the fixture, the documented example, the sandbox credential - the + * things a baseline would also accept, except that the reason travels with + * the code instead of living in a JSON file nobody reads. + */ + public const ALLOW_MARKER = 'redactor:allow'; + public function __construct( - protected Redactor $redactor + protected Redactor $redactor, + protected int $windowLines = LineWindowReader::DEFAULT_WINDOW_LINES, + protected int $overlapLines = LineWindowReader::DEFAULT_OVERLAP_LINES, + /** Verifies while the raw value is in hand; only the verdict reaches a ScanFinding, never the secret. */ + protected ?SecretVerifier $verifier = null, + /** Whether to look one layer deep inside base64, URL-encoded and JSON-escaped spans. */ + protected bool $decode = true, ) {} - public function scanFile(string $filePath, ?string $profile = null): ScanResult + public function withVerifier(?SecretVerifier $verifier): self { - $content = @file_get_contents($filePath); + return new self($this->redactor, $this->windowLines, $this->overlapLines, $verifier, $this->decode); + } - if ($content === false) { + /** + * Scan a file, a window of lines at a time. + * + * Streaming is unconditional rather than only for large files: a code path + * that runs only on the rare large input is the one that ends up buggy. + */ + public function scanFile(string $filePath, ?string $profile = null, ?string $relativeTo = null): ScanResult + { + if (! is_readable($filePath) || ! is_file($filePath)) { return new ScanResult( path: $filePath, findings: [], @@ -26,47 +63,286 @@ public function scanFile(string $filePath, ?string $profile = null): ScanResult ); } - $redacted = $this->redactor->redact($content, $profile); + $reportedPath = $relativeTo !== null + ? $this->relativePath($filePath, $relativeTo) + : $filePath; - /** @var array> $findings */ - $findings = []; + return $this->scanWindows( + new LineWindowReader($filePath, $this->windowLines, $this->overlapLines), + $filePath, + $reportedPath, + $profile + ); + } + + /** + * Scan text held in memory as though it were a file at the given path. + */ + public function scanText(string $content, string $path, ?string $profile = null): ScanResult + { + return $this->scanWindows( + LineWindowReader::ofString($content, $this->windowLines, $this->overlapLines), + $path, + $path, + $profile + ); + } - // Check for array-based redaction (structured data like JSON) - if (is_array($redacted) && isset($redacted['_redacted']) && $redacted['_redacted'] === true) { - /** @var array> $findings */ - $findings = $redacted['_redacted_keys'] ?? []; + /** + * Scan the lines a change added, reporting each finding at its real line and commit. + * + * The added lines are scanned as one text so a secret spanning two adjacent + * added lines is still found; line numbers are then mapped back through the patch. + */ + public function scanPatch(Patch $patch, ?string $profile = null): ScanResult + { + $result = $this->scanText($patch->text(), $patch->path, $profile); + + if (! $result->hasFindings()) { + return $result; } - // Check for string-based redaction (plain text content) - elseif (is_string($redacted) && $redacted !== $content) { - $findings = $this->analyzeStringRedaction($content, $redacted, $profile ?? 'default'); + + return new ScanResult( + path: $result->path, + findings: array_map( + fn (ScanFinding $finding): ScanFinding => $finding->at($patch->lineAt($finding->line), $patch->commit), + $result->findings + ), + profile: $result->profile, + ); + } + + private function scanWindows(LineWindowReader $reader, string $filePath, string $reportedPath, ?string $profile): ScanResult + { + $profileName = $profile ?? 'default'; + + /** @var array $findings */ + $findings = []; + + foreach ($reader as [$startLine, $window]) { + $result = $this->redactor->inspect($window, $profile); + + $located = $this->located($window, $result->value, $result->findings, $reportedPath, $profileName); + + if ($this->decode) { + foreach (Decoder::derive($window) as $derived) { + foreach ($this->locatedInDerived($derived, $window, $reportedPath, $profile, $profileName) as $finding) { + $located[] = $finding; + } + } + } + + foreach ($located as $finding) { + $absolute = $finding->at($startLine + $finding->line - 1, null); + + // Overlapping windows see the same span twice; identity is the rule and the place... + $findings[$absolute->rule.'|'.$absolute->line.'|'.$absolute->column] = $absolute; + } } + $ordered = array_values($findings); + + usort($ordered, fn (ScanFinding $a, ScanFinding $b): int => [$a->line, $a->column] <=> [$b->line, $b->column]); + return new ScanResult( path: $filePath, - findings: $findings, - profile: $profile ?? 'default' + findings: $ordered, + profile: $profileName ); } /** - * Analyze differences between original and redacted string content. + * Locate and verify one window's matches. + * + * @param array $matches + * @return array + */ + private function located(string $window, mixed $redacted, array $matches, string $path, string $profileName): array + { + $verdicts = $this->verifyAll($matches); + $findings = []; + + foreach ($this->locate($window, $redacted, $matches, $path, $profileName) as $index => $finding) { + $findings[] = isset($verdicts[$index]) ? $finding->withVerification($verdicts[$index]) : $finding; + } + + return $findings; + } + + /** + * Scan text recovered from an encoded span, reporting findings at the span's own position. * - * @return array> + * The excerpt comes from the decoded, redacted text so the report shows + * what was found without repeating it. + * + * @return array */ - protected function analyzeStringRedaction(string $original, string $redacted, string $profile): array + private function locatedInDerived(DerivedSubject $derived, string $window, string $path, ?string $profile, string $profileName): array { + $result = $this->redactor->inspect($derived->text, $profile); + + if ($result->findings === []) { + return []; + } + + $lineStarts = $this->lineStarts($window); + $line = $this->lineForOffset($lineStarts, $derived->offset); + $column = $derived->offset - $lineStarts[$line - 1] + 1; + $verdicts = $this->verifyAll($result->findings); + $redacted = is_string($result->value) ? $result->value : ''; + $findings = []; - // Current redaction strategies replace the entire content when any sensitive data is found - if ($redacted === '[REDACTED]') { - $findings[] = [ - 'type' => 'full_content_redacted', - 'reason' => 'Entire content was redacted', - 'original_length' => strlen($original), - 'profile' => $profile, - ]; + foreach ($result->findings as $index => $match) { + $finding = new ScanFinding( + path: $path, + rule: $match->rule, + line: $line, + column: $column, + excerpt: sprintf('[%s] %s', $derived->encoding, $this->excerpt(strtok($redacted, "\n") ?: '')), + profile: $profileName, + fingerprint: ScanFinding::fingerprint($match->rule, $path, $match->matched), + entity: $match->entity(), + confidence: $match->confidence?->score, + signals: $match->confidence?->explain() ?? [], + encoding: $derived->encoding, + ); + + $findings[] = isset($verdicts[$index]) ? $finding->withVerification($verdicts[$index]) : $finding; } return $findings; } + + /** + * Verify each detection, if anything is permitted to. + * + * Keyed by position so the verdict lands on the right finding without the + * secret having to travel alongside it. + * + * @param array $matches + * @return array + */ + protected function verifyAll(array $matches): array + { + if (! $this->verifier instanceof SecretVerifier) { + return []; + } + + $verdicts = []; + + foreach ($matches as $index => $match) { + if ($match->matched === '' || ! $this->verifier->canVerify($match->entity(), $match->rule)) { + continue; + } + + $verdicts[$index] = $this->verifier->verify($match->entity(), $match->rule, $match->matched); + } + + return $verdicts; + } + + /** + * Turn the byte offsets of the matches into file positions. + * + * @param array $matches + * @return array + */ + protected function locate(string $original, mixed $redacted, array $matches, string $path, string $profile): array + { + if ($matches === []) { + return []; + } + + $lineStarts = $this->lineStarts($original); + + // Replacements never add or remove newlines, so line N of the redacted output + // is line N of the input, which is what lets the excerpt come from it... + $redactedLines = is_string($redacted) ? explode("\n", $redacted) : []; + $originalLines = str_contains($original, self::ALLOW_MARKER) ? explode("\n", $original) : null; + + $findings = []; + + foreach ($matches as $match) { + $line = $this->lineForOffset($lineStarts, $match->offset); + $column = $match->offset - $lineStarts[$line - 1] + 1; + + if ($originalLines !== null && str_contains($originalLines[$line - 1] ?? '', self::ALLOW_MARKER)) { + continue; + } + + $findings[] = new ScanFinding( + path: $path, + rule: $match->rule, + line: $line, + column: $column, + excerpt: $this->excerpt($redactedLines[$line - 1] ?? ''), + profile: $profile, + fingerprint: ScanFinding::fingerprint($match->rule, $path, $match->matched), + entity: $match->entity(), + confidence: $match->confidence?->score, + signals: $match->confidence?->explain() ?? [], + ); + } + + return $findings; + } + + /** + * Get the byte offset at which each line begins. + * + * @return array + */ + private function lineStarts(string $content): array + { + $starts = [0]; + $offset = 0; + + while (($position = strpos($content, "\n", $offset)) !== false) { + $starts[] = $position + 1; + $offset = $position + 1; + } + + return $starts; + } + + /** + * @param array $lineStarts + */ + private function lineForOffset(array $lineStarts, int $offset): int + { + $low = 0; + $high = count($lineStarts) - 1; + + while ($low < $high) { + $mid = intdiv($low + $high + 1, 2); + + if ($lineStarts[$mid] <= $offset) { + $low = $mid; + } else { + $high = $mid - 1; + } + } + + return $low + 1; + } + + private function excerpt(string $line): string + { + $line = trim(str_replace(["\r", "\t"], ['', ' '], $line)); + + if (strlen($line) <= self::EXCERPT_LIMIT) { + return $line; + } + + return substr($line, 0, self::EXCERPT_LIMIT).'...'; + } + + private function relativePath(string $path, string $base): string + { + $base = rtrim(realpath($base) ?: $base, '/').'/'; + $real = realpath($path) ?: $path; + + return str_starts_with($real, $base) ? substr($real, strlen($base)) : $real; + } } diff --git a/src/Strategies/BlockedKeysStrategy.php b/src/Strategies/BlockedKeysStrategy.php index 1c1c8bd..1222466 100644 --- a/src/Strategies/BlockedKeysStrategy.php +++ b/src/Strategies/BlockedKeysStrategy.php @@ -4,46 +4,75 @@ namespace Kirschbaum\Redactor\Strategies; +use Kirschbaum\Redactor\Detection\Confidence; +use Kirschbaum\Redactor\Detection\Detection; +use Kirschbaum\Redactor\Operators\OperatorRegistry; use Kirschbaum\Redactor\RedactionContext; +use Kirschbaum\Redactor\Strategies\Contracts\Strategy; -class BlockedKeysStrategy implements RedactionStrategyInterface +/** + * Redacts a value because of the name of the key holding it. + * + * Supports exact names and '*' wildcards ('*token*', 'password*', '*_key', + * 'user_*_token'), compared case-insensitively. The key is the entity, so + * `operators.email` applies to a value under a key named `email` whether the + * key rule or the email pattern found it first. + */ +class BlockedKeysStrategy implements Strategy { + /** + * The certain score shared by every key-based detection. + * + * Built once: this runs for every blocked value in every payload, and a + * fresh Confidence with a formatted reason per value was measurable. + */ + private static ?Confidence $certain = null; + + /** + * Determine if the key is in the profile's blocked list. + */ public function shouldHandle(mixed $value, string $key, RedactionContext $context): bool { - $keyLower = strtolower($key); - - foreach ($context->config->blockedKeys as $blockedKey) { - // Check for wildcard patterns - if ($this->matchesPattern($keyLower, $blockedKey)) { - return true; - } - } - - return false; + // onError: true, since an unevaluatable blocked-key pattern blocks the key... + return $context->wants($key) && $context->config->blockedKeyMatcher->matches($key, onError: true); } + /** + * Redact the value under the blocked key. + */ public function handle(mixed $value, string $key, RedactionContext $context): mixed { - $context->addRedactedKey($key); + $scalar = is_string($value) || is_int($value) || is_float($value); - return $context->config->replacement; - } + if ($scalar && $context->isAllowed((string) $value)) { + return $value; + } - /** - * Check if a key matches a blocked key pattern. - * Supports wildcard patterns using '*' as a wildcard character. - */ - private function matchesPattern(string $key, string $pattern): bool - { - // If no wildcards, do exact match (case-insensitive) - if (strpos($pattern, '*') === false) { - return $key === strtolower($pattern); + $detection = new Detection( + entity: strtolower($key), + rule: 'blocked_key', + offset: 0, + value: $scalar ? (string) $value : '', + confidence: self::$certain ??= Confidence::of(Confidence::CERTAIN, 'the key is in blocked_keys'), + key: $key, + ); + + // Nullify keeps the key and drops the value, since it is the operator for a typed field that must stay a field... + if ($context->operatorSpecFor($detection)->name === OperatorRegistry::NULLIFY) { + $context->recordDetection($detection); + + return null; + } + + // Containers, booleans and nulls have no text an operator could act on, so they collapse to the replacement string... + if (! $scalar) { + $context->recordRedaction($key, 'blocked_key'); + + return $context->config->replacement; } - // Convert wildcard pattern to regex - // Escape the pattern first, then replace escaped wildcards - $regexPattern = '/^'.str_replace('\\*', '.*', preg_quote($pattern, '/')).'$/i'; + $context->recordDetection($detection); - return preg_match($regexPattern, $key) === 1; + return $context->operate($detection); } } diff --git a/src/Strategies/Contracts/ChainableStrategy.php b/src/Strategies/Contracts/ChainableStrategy.php new file mode 100644 index 0000000..ef7742e --- /dev/null +++ b/src/Strategies/Contracts/ChainableStrategy.php @@ -0,0 +1,16 @@ +config->recognition; + + if (($settings['batch'] ?? true) === false) { + return; + } + + $texts = []; + $this->gather($content, '', $context, $texts); + + if ($texts === []) { + return; + } + + $recognizer = $this->recognizer($settings, $context); + + if (! $recognizer instanceof Recognizer || ! CircuitBreaker::allows($this->breakerKey($recognizer, $context))) { + return; + } + + $texts = array_values($texts); + $spans = $this->askMany($recognizer, $texts, $context); + + $primed = []; + + foreach ($texts as $index => $text) { + $primed[$text] = $spans[$index] ?? []; + } + + $context->primeRecognition($primed); + } + + /** + * Ask the recogniser about every text, in one call if it can take a list. + * + * A throw becomes "rules only, this time" for every text in the batch. + * + * @param array $texts + * @return array> + */ + private function askMany(Recognizer $recognizer, array $texts, RedactionContext $context): array + { + $settings = $context->config->recognition; + $language = $this->string($settings, 'language', 'en'); + $entities = $this->labels($settings); + $threshold = $this->float($settings, 'score_threshold', 0.6); + + try { + if ($recognizer instanceof BatchRecognizer) { + $spans = $recognizer->recognizeMany($texts, $language, $entities, $threshold); + } else { + $spans = []; + + foreach ($texts as $index => $text) { + $spans[$index] = $recognizer->recognize($text, $language, $entities, $threshold); + } + } + } catch (Throwable $e) { + $this->failed($recognizer, $context, $e); + + return []; + } + + CircuitBreaker::recordSuccess($this->breakerKey($recognizer, $context)); + + return $spans; + } + + /** + * Collect every prose value the walk would send, keyed by the text itself. + * + * Objects are opened the way the walk opens them, and the same depth and + * cycle guards apply, so a payload the walk would stop on stops here too. + * + * @param array $texts + */ + private function gather(mixed $value, string $key, RedactionContext $context, array &$texts): void + { + if (is_string($value)) { + if (! isset($texts[$value]) && $this->walkWouldRead($key, $context) && $this->shouldHandle($value, $key, $context)) { + $texts[$value] = $value; + } + + return; + } + + if (! $this->walkWouldRead($key, $context)) { + return; + } + + if (is_array($value)) { + $this->gatherFrom($value, $context, $texts); + + return; + } + + // The object stays on the stack while its children are read, so a + // toArray() that hands back its owner is entered once... + if (is_object($value) && $context->enterObject($value)) { + try { + $opened = $this->open($value); + + if ($opened !== null) { + $this->gatherFrom($opened, $context, $texts); + } + } finally { + $context->leaveObject($value); + } + } + } + + /** + * Collect prose from every child of the array, one level deeper. + * + * @param array $array + * @param array $texts + */ + private function gatherFrom(array $array, RedactionContext $context, array &$texts): void + { + if (! $context->enterDepth()) { + return; + } + + try { + foreach ($array as $childKey => $child) { + $this->gather($child, (string) $childKey, $context, $texts); + } + } finally { + $context->leaveDepth(); + } + } + + /** + * Determine if the walk would read the value under the key rather than settle it by name. + */ + private function walkWouldRead(string $key, RedactionContext $context): bool + { + if ($key === '') { + return true; + } + + return ! $context->config->safeKeyMatcher->matches($key, onError: false) + && ! $context->config->blockedKeyMatcher->matches($key); + } + + /** + * Open an object into an array the way the walk does, or null when it is opaque. + * + * @return array|null + */ + private function open(object $object): ?array + { + if ($object instanceof Throwable || $object instanceof \DateTimeInterface || $object instanceof \DateTimeZone || $object instanceof \UnitEnum || $object instanceof \Closure) { + return null; + } + + try { + if (method_exists($object, 'toArray')) { + $array = $object->toArray(); + + return is_array($array) ? $array : null; + } + + $decoded = json_decode(json_encode($object, JSON_THROW_ON_ERROR), true, 512, JSON_THROW_ON_ERROR); + + return is_array($decoded) ? $decoded : null; + } catch (Throwable) { + return null; + } + } + + /** + * Determine if the profile enables entity recognition. + */ + public function appliesTo(RedactorConfig $config): bool + { + return ($config->recognition['enabled'] ?? false) === true; + } + + /** + * Determine if the value is prose the recogniser should read. + */ + public function shouldHandle(mixed $value, string $key, RedactionContext $context): bool + { + if (! is_string($value)) { + return false; + } + + $settings = $context->config->recognition; + + if (($settings['enabled'] ?? false) !== true) { + return false; + } + + $length = strlen($value); + + if ($length < $this->int($settings, 'min_length', 20) || $length > $this->int($settings, 'max_length', 5000)) { + return false; + } + + return $this->looksLikeProse($value, $this->int($settings, 'min_words', 3)); + } + + /** + * Collect every entity the recogniser finds in the value. + */ + public function handle(mixed $value, string $key, RedactionContext $context): mixed + { + if (! is_string($value)) { + return $value; + } + + foreach ($this->detect($value, $key, $context) as $detection) { + $context->collect($detection); + } + + return $value; + } + + /** + * Get every entity the recogniser finds in the subject. + * + * @return array + */ + public function detect(string $subject, string $key, RedactionContext $context): array + { + $settings = $context->config->recognition; + $recognizer = $this->recognizer($settings, $context); + + if (! $recognizer instanceof Recognizer) { + return []; + } + + $threshold = $this->float($settings, 'score_threshold', 0.6); + $spans = $context->primedRecognition($subject); + + if ($spans === null) { + if (! CircuitBreaker::allows($this->breakerKey($recognizer, $context))) { + return []; + } + + $spans = $this->askOne($recognizer, $subject, $context); + } + + return $this->toDetections($spans, $subject, $key, $recognizer->name(), $settings, $threshold); + } + + /** + * Ask the recogniser about one text. + * + * A throw becomes "rules only, this time" for that text. + * + * @return array + */ + private function askOne(Recognizer $recognizer, string $subject, RedactionContext $context): array + { + $settings = $context->config->recognition; + + try { + $spans = $recognizer->recognize( + $subject, + $this->string($settings, 'language', 'en'), + $this->labels($settings), + $this->float($settings, 'score_threshold', 0.6) + ); + } catch (Throwable $e) { + $this->failed($recognizer, $context, $e); + + return []; + } + + CircuitBreaker::recordSuccess($this->breakerKey($recognizer, $context)); + + return $spans; + } + + /** + * Count a failed call against the breaker and log it through the re-entrancy guard. + */ + private function failed(Recognizer $recognizer, RedactionContext $context, Throwable $e): void + { + $settings = $context->config->recognition; + + $opened = CircuitBreaker::recordFailure( + $this->breakerKey($recognizer, $context), + $this->int($settings, 'failure_threshold', 3), + $this->int($settings, 'cooldown', 60) + ); + + InternalLog::warning('Entity recognition failed; continuing with rules only', [ + 'recognizer' => $recognizer->name(), + 'profile' => $context->config->profile, + 'reason' => $e->getMessage(), + 'breaker_opened' => $opened, + ]); + } + + /** + * Get the breaker key for the recogniser under this profile. + */ + private function breakerKey(Recognizer $recognizer, RedactionContext $context): string + { + return $recognizer->name().'|'.$context->config->profile; + } + + /** + * Convert recognised spans into detections, verifying every offset. + * + * @param array $spans + * @param array $settings + * @return array + */ + private function toDetections(array $spans, string $subject, string $key, string $recognizer, array $settings, float $threshold): array + { + $map = $this->entityMap($settings); + $wanted = $this->labels($settings); + $characters = mb_strlen($subject, 'UTF-8'); + $detections = []; + + foreach ($spans as $span) { + if ($span->score < $threshold) { + continue; + } + + if ($span->end <= $span->start || $span->start < 0 || $span->end > $characters) { + $this->warnMisaligned($recognizer, $span); + + continue; + } + + if ($wanted !== [] && ! in_array($span->entity, $wanted, true)) { + continue; + } + + $byteOffset = strlen(mb_substr($subject, 0, $span->start, 'UTF-8')); + $value = mb_substr($subject, $span->start, $span->end - $span->start, 'UTF-8'); + + // The recogniser tokenised its own copy of the text, so a span whose offsets + // do not land on the same characters here is skipped rather than guessed at... + if (trim($value) === '' || substr($subject, $byteOffset, strlen($value)) !== $value) { + $this->warnMisaligned($recognizer, $span); + + continue; + } + + $entity = $map[$span->entity] ?? strtolower($span->entity); + + $confidence = Confidence::of( + $span->score, + sprintf('recognised as %s by %s', $span->entity, $recognizer) + ); + + $detections[] = new Detection( + entity: $entity, + rule: self::RULE, + offset: $byteOffset, + value: $value, + confidence: KeywordContext::boost($confidence, $subject, $byteOffset, $key), + key: $key, + ); + } + + return $detections; + } + + /** + * Log a span whose offsets do not align with the subject. + */ + private function warnMisaligned(string $recognizer, RecognizedSpan $span): void + { + InternalLog::warning('Entity recognition returned a span that does not align with the subject; skipped', [ + 'recognizer' => $recognizer, + 'entity' => $span->entity, + 'start' => $span->start, + 'end' => $span->end, + ]); + } + + /** + * Resolve the configured recogniser, if it is registered. + * + * @param array $settings + */ + private function recognizer(array $settings, RedactionContext $context): ?Recognizer + { + $driver = $this->string($settings, 'driver', 'presidio'); + $recognizer = $context->recognizers()->get($driver); + + if (! $recognizer instanceof Recognizer) { + InternalLog::warning('Unknown entity recogniser; continuing with rules only', [ + 'driver' => $driver, + 'available' => $context->recognizers()->names(), + 'profile' => $context->config->profile, + ]); + + return null; + } + + if ($recognizer instanceof PresidioRecognizer && isset($settings['url']) && is_string($settings['url'])) { + return $recognizer->withEndpoint($settings['url'], $this->float($settings, 'timeout', 2.0)); + } + + return $recognizer; + } + + /** + * Determine if a value reads like text a model was trained on. + * + * A JSON document, a stack trace or a single token does not; the model + * would guess, and its guesses are the false positives this gate avoids. + */ + protected function looksLikeProse(string $value, int $minWords): bool + { + $trimmed = ltrim($value); + + if ($trimmed === '' || $trimmed[0] === '{' || $trimmed[0] === '[' || $trimmed[0] === '<') { + return false; + } + + $words = preg_split('/\s+/', trim($value), -1, PREG_SPLIT_NO_EMPTY); + + if ($words === false || count($words) < max(1, $minWords)) { + return false; + } + + $wordy = 0; + + foreach ($words as $word) { + if (preg_match('/^[\p{L}][\p{L}\p{M}\'’.,;:!?-]*$/u', $word) === 1) { + $wordy++; + } + } + + return $wordy * 2 >= count($words); + } + + /** + * Get the entity labels the profile asks for. + * + * @param array $settings + * @return array + */ + private function labels(array $settings): array + { + $entities = $settings['entities'] ?? []; + + return is_array($entities) ? array_values(array_filter($entities, is_string(...))) : []; + } + + /** + * Get the map from recogniser labels to package entities. + * + * @param array $settings + * @return array + */ + private function entityMap(array $settings): array + { + $map = $settings['entity_map'] ?? []; + + if (! is_array($map)) { + return []; + } + + $out = []; + + foreach ($map as $label => $entity) { + if (is_string($label) && is_string($entity) && $entity !== '') { + $out[$label] = $entity; + } + } + + return $out; + } + + /** + * Get an integer setting, or the default. + * + * @param array $settings + */ + private function int(array $settings, string $key, int $default): int + { + $value = $settings[$key] ?? null; + + return is_numeric($value) ? (int) $value : $default; + } + + /** + * Get a float setting, or the default. + * + * @param array $settings + */ + private function float(array $settings, string $key, float $default): float + { + $value = $settings[$key] ?? null; + + return is_numeric($value) ? (float) $value : $default; + } + + /** + * Get a non-empty string setting, or the default. + * + * @param array $settings + */ + private function string(array $settings, string $key, string $default): string + { + $value = $settings[$key] ?? null; + + return is_string($value) && $value !== '' ? $value : $default; + } +} diff --git a/src/Strategies/KnownSecretsStrategy.php b/src/Strategies/KnownSecretsStrategy.php new file mode 100644 index 0000000..d9ad1ea --- /dev/null +++ b/src/Strategies/KnownSecretsStrategy.php @@ -0,0 +1,72 @@ +secrets()->couldContainOne($value); + } + + /** + * Collect every registered secret found in the value. + */ + public function handle(mixed $value, string $key, RedactionContext $context): mixed + { + if (! is_string($value)) { + return $value; + } + + foreach ($this->detect($value, $key, $context) as $detection) { + $context->collect($detection); + } + + return $value; + } + + /** + * Get every occurrence of a registered secret in the subject. + * + * @return array + */ + public function detect(string $subject, string $key, RedactionContext $context): array + { + $detections = []; + + foreach ($context->secrets()->find($subject) as $hit) { + $detections[] = new Detection( + entity: $hit['entity'], + rule: self::RULE, + offset: $hit['offset'], + value: $hit['value'], + confidence: Confidence::of(Confidence::CERTAIN, 'a registered secret value appears verbatim'), + key: $key, + ); + } + + return $detections; + } +} diff --git a/src/Strategies/LargeObjectStrategy.php b/src/Strategies/LargeObjectStrategy.php index 2546c28..bea2163 100644 --- a/src/Strategies/LargeObjectStrategy.php +++ b/src/Strategies/LargeObjectStrategy.php @@ -5,37 +5,46 @@ namespace Kirschbaum\Redactor\Strategies; use Kirschbaum\Redactor\RedactionContext; +use Kirschbaum\Redactor\Strategies\Contracts\Strategy; -class LargeObjectStrategy implements RedactionStrategyInterface +/** + * Replaces a container with more items than the profile allows. + */ +class LargeObjectStrategy implements Strategy { + /** + * Determine if the value has more items than the profile allows. + */ public function shouldHandle(mixed $value, string $key, RedactionContext $context): bool { - if (! $context->config->redactLargeObjects) { + $maxObjectSize = $context->config->maxObjectSize; + + if (! $context->config->redactLargeObjects || $maxObjectSize === null) { return false; } if (is_array($value)) { - return count($value) > $context->config->maxObjectSize; + return count($value) > $maxObjectSize; } if (is_object($value)) { - // For objects, we'll need to check if they can be converted to array first + // Objects are measured through toArray() when they offer it... if (method_exists($value, 'toArray')) { try { $array = $value->toArray(); - return is_array($array) && count($array) > $context->config->maxObjectSize; + return is_array($array) && count($array) > $maxObjectSize; } catch (\Throwable) { return false; } } - // Try JSON encoding to get a size estimate + // Otherwise estimate the size through a JSON round trip... try { $jsonString = json_encode($value, JSON_THROW_ON_ERROR); $array = json_decode($jsonString, true, 512, JSON_THROW_ON_ERROR); - return is_array($array) && count($array) > $context->config->maxObjectSize; + return is_array($array) && count($array) > $maxObjectSize; } catch (\Throwable) { return false; } @@ -44,6 +53,9 @@ public function shouldHandle(mixed $value, string $key, RedactionContext $contex return false; } + /** + * Replace the value with a summary of what it held. + */ public function handle(mixed $value, string $key, RedactionContext $context): mixed { $context->markRedacted(); @@ -59,7 +71,7 @@ public function handle(mixed $value, string $key, RedactionContext $context): mi } if (is_object($value)) { - // Try to get property count for more accurate messaging + // Count the properties for the message where the object allows it... $propertyCount = 'large number of'; try { if (method_exists($value, 'toArray')) { @@ -75,14 +87,14 @@ public function handle(mixed $value, string $key, RedactionContext $context): mi } } } catch (\Throwable) { - // Keep default message + // Keep the default message... } return [ '_large_object_redacted' => sprintf( '%s (Object %s with %s properties)', $context->config->replacement, - get_class($value), + $value::class, $propertyCount ), ]; diff --git a/src/Strategies/LargeStringStrategy.php b/src/Strategies/LargeStringStrategy.php index bea0a3a..75110f6 100644 --- a/src/Strategies/LargeStringStrategy.php +++ b/src/Strategies/LargeStringStrategy.php @@ -5,9 +5,24 @@ namespace Kirschbaum\Redactor\Strategies; use Kirschbaum\Redactor\RedactionContext; +use Kirschbaum\Redactor\Strategies\Contracts\ChainableStrategy; +use Kirschbaum\Redactor\Strategies\Contracts\Strategy; -class LargeStringStrategy implements RedactionStrategyInterface +/** + * Bounds the work done on a very long string. + * + * max_value_length exists so a pathological value, a multi-megabyte blob or a + * base64 image, cannot make one log line cost seconds. Replacing the whole + * value threw away the thing most often over the limit in a Laravel log, a + * stack trace or a request body, so the default keeps the head, marks what was + * cut, and hands the head on so a secret in it is still found. + * `large_string_behavior: redact` restores the wholesale replacement. + */ +class LargeStringStrategy implements ChainableStrategy, Strategy { + /** + * Determine if the string exceeds the profile's maximum length. + */ public function shouldHandle(mixed $value, string $key, RedactionContext $context): bool { return is_string($value) @@ -15,14 +30,39 @@ public function shouldHandle(mixed $value, string $key, RedactionContext $contex && strlen($value) > $context->config->maxValueLength; } + /** + * Truncate or replace the string according to the profile. + */ public function handle(mixed $value, string $key, RedactionContext $context): mixed { - $context->markRedacted(); - if (! is_string($value)) { + $context->markRedacted(); + return $value; } - return sprintf('%s (String with %d characters)', $context->config->replacement, strlen($value)); + $length = strlen($value); + $replacement = $context->config->replacement; + + if ($context->config->largeStringBehavior === 'redact') { + $context->recordRedaction($key, 'large_string', 0, $length); + + return sprintf('%s (String with %d characters)', $replacement, $length); + } + + $limit = $context->config->maxValueLength ?? $length; + + // mb_strcut never splits a multibyte sequence, so the head stays valid UTF-8 for the strategies that scan it next... + $head = mb_strcut($value, 0, $limit, 'UTF-8'); + + $context->recordRedaction($key, 'large_string', strlen($head), $length - strlen($head)); + + return sprintf( + '%s %s (String truncated: %d characters, %d kept)', + $head, + $replacement, + $length, + strlen($head) + ); } } diff --git a/src/Strategies/RedactionStrategyInterface.php b/src/Strategies/RedactionStrategyInterface.php deleted file mode 100644 index 4d1d1b1..0000000 --- a/src/Strategies/RedactionStrategyInterface.php +++ /dev/null @@ -1,18 +0,0 @@ -config->patterns)) { - return false; + return is_string($value) && $context->config->patterns !== []; + } + + /** + * Collect every pattern match in the value. + */ + public function handle(mixed $value, string $key, RedactionContext $context): mixed + { + if (! is_string($value)) { + return $value; } - foreach ($context->config->patterns as $pattern) { - if (preg_match($pattern, $value)) { - return true; + foreach ($this->detect($value, $key, $context) as $detection) { + $context->collect($detection); + } + + return $value; + } + + /** + * Get every span any rule accepts, in the order the rules are configured. + * + * Overlaps between rules are left in; the context resolves them. + * + * @return array + */ + public function detect(string $subject, string $key, RedactionContext $context): array + { + $detections = []; + $lowered = null; + $length = strlen($subject); + + foreach ($context->config->patternsByLength as [$rule, $priority]) { + // A rule whose shortest possible match is longer than the subject cannot + // match it, and neither can any rule after it in this length order... + if ($rule->minLength > $length) { + break; + } + + // A rule that names keywords only runs on a subject containing one, checked + // with str_contains() per rule since a thirty-way alternation costs PCRE + // more than every rule it was meant to save... + if ($rule->keywords !== []) { + $lowered ??= strtolower($subject); + + if (! $this->containsAny($lowered, $rule->keywords)) { + continue; + } + } + + // A capture-free preg_match() on a non-matching subject costs a fraction of + // preg_match_all() with offsets, and most rules do not match most values... + if (@preg_match($rule->pattern, $subject) === 0) { + continue; + } + + // A pre-check the engine could not finish falls through here and fails the same way... + $found = $this->detectRule($rule, $subject, $key, $priority); + + if ($found === null) { + // The engine gave up partway through, and a partially inspected string + // would leak whatever it did not reach, so report all of it... + Pcre::matches($rule->pattern, $subject, onError: true, rule: $rule->name); + + return [Detection::failClosed( + $rule->entity(), + $rule->name, + $subject, + $key, + sprintf('pattern "%s" could not be evaluated; failing closed', $rule->name) + )]; + } + + foreach ($found as $detection) { + $detections[] = $detection; } } - return false; + return $detections; } - public function handle(mixed $value, string $key, RedactionContext $context): mixed + /** + * Get every span in the subject one rule accepts, in order. + * + * Returns null if the engine failed; an empty array means a clean subject. + * + * @return array|null + */ + private function detectRule(PatternRule $rule, string $subject, string $key, int $priority): ?array { - $context->markRedacted(); + $found = @preg_match_all($rule->pattern, $subject, $matches, PREG_SET_ORDER | PREG_OFFSET_CAPTURE); - return $context->config->replacement; + if ($found === false || preg_last_error() !== PREG_NO_ERROR) { + return null; + } + + $operator = $rule->hasExplicitOperator() ? $rule->operatorSpec() : null; + $detections = []; + + foreach ($matches as $set) { + $target = $rule->capture > 0 && isset($set[$rule->capture]) && $set[$rule->capture][1] >= 0 + ? $set[$rule->capture] + : $set[0]; + + [$text, $offset] = [(string) $target[0], (int) $target[1]]; + + if ($text === '' || ! $rule->accepts($text)) { + continue; + } + + $confidence = $this->score($rule, $subject, $offset, $key); + + if ($rule->replacesWholeValue()) { + // Legacy full mode condemns the entire value on one match, and a span + // the width of the subject swallows every other report... + return [new Detection( + entity: $rule->entity(), + rule: $rule->name, + offset: 0, + value: $subject, + confidence: $confidence, + key: $key, + operator: $operator, + priority: $priority, + )]; + } + + $detections[] = new Detection( + entity: $rule->entity(), + rule: $rule->name, + offset: $offset, + value: $text, + confidence: $confidence, + key: $key, + operator: $operator, + priority: $priority, + ); + } + + return $detections; + } + + /** + * Score a match from the rule's base confidence plus what surrounds it. + */ + private function score(PatternRule $rule, string $subject, int $offset, string $key): Confidence + { + $confidence = Confidence::of($rule->confidence, sprintf('pattern "%s" matched', $rule->name)); + + if ($rule->validator !== null) { + $confidence = $confidence->with( + 'validator', + self::VALIDATOR_BOOST, + sprintf('%s checksum passed', $rule->validator) + ); + } + + return KeywordContext::boost($confidence, $subject, $offset, $key); + } + + /** + * Determine if the haystack contains any of the given needles. + * + * @param array $needles already lowercased + */ + private function containsAny(string $haystack, array $needles): bool + { + foreach ($needles as $needle) { + if (str_contains($haystack, $needle)) { + return true; + } + } + + return false; } } diff --git a/src/Strategies/SafeKeysStrategy.php b/src/Strategies/SafeKeysStrategy.php index cfd40d2..e1bb989 100644 --- a/src/Strategies/SafeKeysStrategy.php +++ b/src/Strategies/SafeKeysStrategy.php @@ -5,19 +5,34 @@ namespace Kirschbaum\Redactor\Strategies; use Kirschbaum\Redactor\RedactionContext; +use Kirschbaum\Redactor\Strategies\Contracts\PreservingStrategy; +use Kirschbaum\Redactor\Strategies\Contracts\Strategy; -class SafeKeysStrategy implements RedactionStrategyInterface +/** + * Declares a value safe by the name of the key holding it. + * + * Everything under a safe key is preserved, nested structures included, so only + * list keys whose contents cannot carry sensitive data by construction: + * identifiers, timestamps, enumerations. A free-text field like "message" is + * not safe just because it usually looks harmless. Supports the same '*' + * wildcards as BlockedKeysStrategy, compared case-insensitively. + */ +class SafeKeysStrategy implements PreservingStrategy, Strategy { + /** + * Determine if the key is in the profile's safe list. + */ public function shouldHandle(mixed $value, string $key, RedactionContext $context): bool { - $keyLower = strtolower($key); - - return in_array($keyLower, $context->config->safeKeys, true); + // onError: false, since a safe-key pattern that cannot be evaluated must not declare the value safe... + return $context->config->safeKeyMatcher->matches($key, onError: false); } + /** + * Return the value untouched. + */ public function handle(mixed $value, string $key, RedactionContext $context): mixed { - // Safe keys are never redacted - return original value return $value; } } diff --git a/src/Strategies/ShannonEntropyStrategy.php b/src/Strategies/ShannonEntropyStrategy.php index 4a84438..fa7127a 100644 --- a/src/Strategies/ShannonEntropyStrategy.php +++ b/src/Strategies/ShannonEntropyStrategy.php @@ -4,10 +4,45 @@ namespace Kirschbaum\Redactor\Strategies; +use Kirschbaum\Redactor\Detection\Confidence; +use Kirschbaum\Redactor\Detection\Detection; +use Kirschbaum\Redactor\Detection\Detector; +use Kirschbaum\Redactor\Detection\KeywordContext; use Kirschbaum\Redactor\RedactionContext; +use Kirschbaum\Redactor\RedactorConfig; +use Kirschbaum\Redactor\Strategies\Contracts\DetectingStrategy; +use Kirschbaum\Redactor\Strategies\Contracts\Strategy; +use Kirschbaum\Redactor\Support\Pcre; -class ShannonEntropyStrategy implements RedactionStrategyInterface +/** + * Finds tokens that look random enough to be a credential. + * + * Reports detections rather than rewriting, like every other detector, so an + * entropy hit is scored, filtered by the confidence floor and handed to the + * configured operator exactly as a pattern match is, and a surrogate the regex + * detector just wrote, which has the same entropy as the value it replaced, is + * never mistaken for a fresh secret. + */ +class ShannonEntropyStrategy implements DetectingStrategy, Detector, Strategy { + public const ENTITY = 'high_entropy'; + + public const RULE = 'shannon_entropy'; + + /** + * The confidence of a bare entropy hit on its own. + * + * Randomness is evidence of a secret, not proof: a base64 image chunk or a + * git hash scores just as high. A detection starts at medium, climbs with + * its margin over the threshold, and reaches high only with a keyword. + */ + private const float BASE_CONFIDENCE = 0.5; + + private const float MARGIN_BOOST_CAP = 0.4; + + /** + * Determine if the value is a string long enough to measure. + */ public function shouldHandle(mixed $value, string $key, RedactionContext $context): bool { $shannonConfig = $context->config->shannonEntropy; @@ -16,67 +51,285 @@ public function shouldHandle(mixed $value, string $key, RedactionContext $contex return false; } - return $this->shouldRedactByEntropy($value, $context); + return ! $this->tooShort($value, $shannonConfig); } + /** + * Collect every high-entropy token in the value. + */ public function handle(mixed $value, string $key, RedactionContext $context): mixed { - $context->markRedacted(); + if (! is_string($value)) { + return $value; + } + + foreach ($this->detect($value, $key, $context) as $detection) { + $context->collect($detection); + } + + return $value; + } + + /** + * Get every whitespace-delimited token whose entropy clears its threshold. + * + * A value with no internal whitespace is a single token, so a bare API key + * is reported whole; a sentence with a secret in it reports only the secret. + * + * @return array + */ + public function detect(string $subject, string $key, RedactionContext $context): array + { + if ($this->tooShort($subject, $context->config->shannonEntropy)) { + return []; + } + + // Only tokens at least min_length long can qualify and a byte count bounds a + // character count, so asking PCRE for `\S{n,}` is exact and turns a million-byte + // subject into a few hundred candidates instead of hundreds of thousands... + $minimum = max(1, $this->minimumLength($context->config->shannonEntropy)); + + // Spelled out rather than selected into a variable so the /u decision is visible where it matters... + $found = $this->isAscii($subject) + ? @preg_match_all('/\S{'.$minimum.',}/', $subject, $matches, PREG_OFFSET_CAPTURE) + : @preg_match_all('/\S{'.$minimum.',}/u', $subject, $matches, PREG_OFFSET_CAPTURE); + + if ($found === false || preg_last_error() !== PREG_NO_ERROR) { + // The engine gave up, so fail closed rather than let a value the tokeniser could not split go out uninspected... + Pcre::matches('/\S+/u', $subject, onError: true, rule: self::RULE); + + return [Detection::failClosed( + self::ENTITY, + self::RULE, + $subject, + $key, + 'the value could not be tokenised; failing closed' + )]; + } + + $detections = []; + + foreach ($matches[0] as [$token, $offset]) { + if ($token === '' || ! $this->shouldRedactByEntropy($token, $context)) { + continue; + } + + $detections[] = new Detection( + entity: self::ENTITY, + rule: self::RULE, + offset: $offset, + value: $token, + confidence: $this->score($token, $subject, $offset, $key, $context), + key: $key, + ); + } + + return $detections; + } + + /** + * Score a token by how far its entropy clears the threshold, plus context. + */ + protected function score(string $token, string $subject, int $offset, string $key, RedactionContext $context): Confidence + { + $entropy = $this->calculateShannonEntropy($token, $context); + $threshold = $this->thresholdFor($token, $context); + + $confidence = Confidence::of( + self::BASE_CONFIDENCE, + sprintf('entropy %.2f bits/char over the %.2f threshold', $entropy, $threshold) + ); + + $margin = min(self::MARGIN_BOOST_CAP, max(0.0, ($entropy - $threshold) / 2)); + + if ($margin > 0.0) { + $confidence = $confidence->with('margin', $margin, 'well clear of the threshold'); + } + + return KeywordContext::boost($confidence, $subject, $offset, $key); + } + + /** + * Determine if a subject is pure ASCII and can use the cheaper patterns. + * + * The /u modifier makes PCRE validate the whole subject as UTF-8 on every + * call, 40us against 12us to split a 2.2KB string, on a path that runs + * over every value. Dropping /u for non-ASCII input would be wrong rather + * than slower, since \s would stop recognising Unicode whitespace. + */ + protected function isAscii(string $value): bool + { + return preg_match('/[\x80-\xff]/', $value) !== 1; + } + + /** + * Determine if a subject is too short to contain anything worth measuring. + * + * A byte count is an upper bound on a character count, so a subject under + * the minimum in bytes is certainly under it in characters, and only what + * survives that test pays for the encoding check. Applied per value as + * well as per token, since most values in a log payload are well under it. + * + * @param array $shannonConfig + */ + protected function tooShort(string $subject, array $shannonConfig): bool + { + $minLength = $this->minimumLength($shannonConfig); + + if (strlen($subject) < $minLength) { + return true; + } + + return $this->length($subject) < $minLength; + } + + /** + * Get the configured minimum token length, or zero when there is none. + * + * @param array $shannonConfig + */ + protected function minimumLength(array $shannonConfig): int + { + $minLength = $shannonConfig['min_length'] ?? 25; + + return is_numeric($minLength) ? max(0, (int) $minLength) : 0; + } + + /** + * Split a string into characters, falling back to bytes for input that is not valid UTF-8. + * + * @return array + */ + protected function characters(string $string): array + { + if (! mb_check_encoding($string, 'UTF-8')) { + return str_split($string); + } - return $context->config->replacement; + $characters = mb_str_split($string, 1, 'UTF-8'); + + return $characters === [] ? str_split($string) : $characters; } /** - * Determine if a string should be redacted based on Shannon entropy. + * Get the character count, or the byte count for non-UTF-8 input. + */ + protected function length(string $string): int + { + return mb_check_encoding($string, 'UTF-8') + ? mb_strlen($string, 'UTF-8') + : strlen($string); + } + + /** + * The charsets a token can be drawn from, most restrictive first. + * + * A 40-character hex digest tops out at 4 bits per character because it + * has only 16 symbols, so judging it against a base64 threshold guarantees + * a miss and the reverse guarantees false positives. detect-secrets solves + * this the same way: pick the threshold from the alphabet. + * + * @var array + */ + protected const CHARSET_PATTERNS = [ + 'hex' => '/^[0-9a-f]+$/i', + 'base64' => '/^[A-Za-z0-9+\/]+={0,2}$/', + 'base64url' => '/^[A-Za-z0-9_-]+$/', + ]; + + /** + * Determine if a token's entropy clears its threshold. */ protected function shouldRedactByEntropy(string $string, RedactionContext $context): bool { $shannonConfig = $context->config->shannonEntropy; - // Only analyze strings that meet minimum length requirement - $minLength = $shannonConfig['min_length'] ?? 25; - if (strlen($string) < $minLength) { + // Counted in characters, not bytes, so a short multibyte token is not mistaken for a long one... + if ($this->tooShort($string, $shannonConfig)) { return false; } - // Skip common words and patterns that might have high entropy but are not sensitive - if ($this->isCommonPattern($string, $context)) { + // Skip configured exclusions that score high without being sensitive... + if ($this->isCommonPattern($string, $context->config)) { return false; } $entropy = $this->calculateShannonEntropy($string, $context); - $threshold = $shannonConfig['threshold'] ?? 4.8; - return $entropy >= $threshold; + return $entropy >= $this->thresholdFor($string, $context); } /** - * Calculate Shannon entropy of a string with caching. + * Get the entropy threshold to judge this particular token against. + * + * charset_thresholds is an opt-in refinement: when a profile configures + * one for the token's alphabet it wins, otherwise the profile's single + * `threshold` applies. An explicit threshold is never silently overridden. */ - protected function calculateShannonEntropy(string $string, RedactionContext $context): float + protected function thresholdFor(string $string, RedactionContext $context): float { - // Check cache first - $cachedEntropy = $context->getCachedEntropy($string); + $shannonConfig = $context->config->shannonEntropy; + + $configured = $shannonConfig['charset_thresholds'] ?? []; + + if (is_array($configured) && $configured !== []) { + $charset = $this->detectCharset($string); + + if ($charset !== null && is_numeric($configured[$charset] ?? null)) { + /** @var numeric $value */ + $value = $configured[$charset]; + + return (float) $value; + } + } + + $fallback = $shannonConfig['threshold'] ?? 4.8; + + return is_numeric($fallback) ? (float) $fallback : 4.8; + } + + /** + * Identify the alphabet a token is drawn from, if it is a recognised one. + */ + protected function detectCharset(string $string): ?string + { + foreach (self::CHARSET_PATTERNS as $name => $pattern) { + if (Pcre::matches($pattern, $string, onError: false, rule: 'charset:'.$name)) { + return $name; + } + } + + return null; + } + + /** + * Calculate the Shannon entropy of a string, in bits per character. + * + * Pass a context to reuse and populate its per-redaction entropy cache. + */ + public function calculateShannonEntropy(string $string, ?RedactionContext $context = null): float + { + $cachedEntropy = $context?->getCachedEntropy($string); if ($cachedEntropy !== null) { return $cachedEntropy; } - $length = strlen($string); + // Measured in characters, since counting UTF-8 continuation bytes as symbols inflates entropy for non-ASCII text... + $characters = $this->characters($string); + $length = count($characters); + if ($length <= 1) { $entropy = 0.0; - $context->cacheEntropy($string, $entropy); + $context?->cacheEntropy($string, $entropy); return $entropy; } - // Count character frequencies and calculate entropy in a single loop $frequencies = []; - for ($i = 0; $i < $length; $i++) { - $char = $string[$i]; + foreach ($characters as $char) { $frequencies[$char] = ($frequencies[$char] ?? 0) + 1; } - // Calculate entropy $entropy = 0.0; foreach ($frequencies as $frequency) { $probability = $frequency / $length; @@ -85,18 +338,17 @@ protected function calculateShannonEntropy(string $string, RedactionContext $con } } - // Cache the result - $context->cacheEntropy($string, $entropy); + $context?->cacheEntropy($string, $entropy); return $entropy; } /** - * Check if a string matches common patterns that shouldn't be redacted despite high entropy. + * Determine if a string matches a configured exclusion pattern and should be left alone. */ - protected function isCommonPattern(string $string, RedactionContext $context): bool + public function isCommonPattern(string $string, RedactorConfig $config): bool { - $shannonConfig = $context->config->shannonEntropy; + $shannonConfig = $config->shannonEntropy; $exclusionPatterns = $shannonConfig['exclusion_patterns'] ?? []; if (! is_array($exclusionPatterns)) { @@ -108,10 +360,11 @@ protected function isCommonPattern(string $string, RedactionContext $context): b continue; } - if (preg_match($pattern, $string)) { - // Special case: hex strings need additional length check + // onError: false, since an exclusion pattern that cannot be evaluated must not excuse the value... + if (Pcre::matches($pattern, $string, onError: false, rule: 'exclusion_pattern')) { + // A long hex string may be a digest such as SHA-256, so the hex exclusion does not excuse it... if ($pattern === '/^[0-9a-f]+$/i' && strlen($string) >= 32) { - continue; // Long hex strings might be sensitive (like SHA256) + continue; } return true; diff --git a/src/Strategies/StrategyOutcome.php b/src/Strategies/StrategyOutcome.php new file mode 100644 index 0000000..6bd4811 --- /dev/null +++ b/src/Strategies/StrategyOutcome.php @@ -0,0 +1,24 @@ +buffer .= $chunk; + + $cut = $this->cutPoint(); + + if ($cut <= 0) { + return ''; + } + + $head = substr($this->buffer, 0, $cut); + $this->buffer = substr($this->buffer, $cut); + + return $this->redact($head); + } + + /** + * Redact and return everything still held once the stream has ended. + */ + public function flush(): string + { + $rest = $this->buffer; + $this->buffer = ''; + + return $rest === '' ? '' : $this->redact($rest); + } + + /** + * Redact an iterable of chunks, yielding output as it becomes safe. + * + * @param iterable $chunks + * @return Generator + */ + public function through(iterable $chunks): Generator + { + foreach ($chunks as $chunk) { + $out = $this->push($chunk); + + if ($out !== '') { + yield $out; + } + } + + $out = $this->flush(); + + if ($out !== '') { + yield $out; + } + } + + /** + * Wrap a callback that echoes its output so what it echoes is redacted on its way out. + * + * @return callable(): void + */ + public function wrap(callable $callback, int $chunkSize = 4096): callable + { + return function () use ($callback, $chunkSize): void { + ob_start(function (string $chunk, int $phase): string { + $out = $this->push($chunk); + + if (($phase & PHP_OUTPUT_HANDLER_FINAL) !== 0) { + $out .= $this->flush(); + } + + return $out; + }, $chunkSize); + + try { + $callback(); + } finally { + ob_end_flush(); + } + }; + } + + /** + * Create a streamed response whose callback's output is redacted as it streams. + * + * @param array> $headers + */ + public function response(callable $callback, int $status = 200, array $headers = [], int $chunkSize = 4096): StreamedResponse + { + return new StreamedResponse($this->wrap($callback, $chunkSize), $status, $headers); + } + + /** + * Get where the buffer can be cut so nothing emitted could be half a secret. + * + * Returns 0 when nothing can be emitted yet. + */ + private function cutPoint(): int + { + $length = strlen($this->buffer); + + if ($length <= $this->holdback) { + return 0; + } + + $limit = $length - $this->holdback; + + // A PEM block is one secret however many lines it spans, so hold it while open and until it can go whole... + $begin = strrpos($this->buffer, '-----BEGIN'); + + if ($begin !== false) { + $end = strpos($this->buffer, '-----END', $begin); + + if ($end === false) { + $limit = min($limit, $begin); + } else { + $endLine = strpos($this->buffer, "\n", $end); + $blockEnd = $endLine === false ? $length : $endLine + 1; + + if ($limit < $blockEnd) { + $limit = min($limit, $begin); + } + } + } + + return $limit <= 0 ? 0 : $this->boundaryBefore($limit); + } + + /** + * Get where before $limit the buffer can be cut without splitting a secret. + * + * A line end is always safe, since no shipped rule except the PEM block + * matches across a newline. A word boundary is not, so it is used only + * once a whole extra window has passed with no line end, and with no + * boundary at all an unbroken run is emitted once it has outlived a window. + */ + private function boundaryBefore(int $limit): int + { + $head = substr($this->buffer, 0, $limit); + $newline = strrpos($head, "\n"); + + if ($newline !== false) { + return $newline + 1; + } + + if ($limit < $this->holdback) { + return 0; + } + + if (preg_match('/\s(?=\S*$)/', $head, $m, PREG_OFFSET_CAPTURE) === 1) { + return (int) $m[0][1] + 1; + } + + return $limit; + } + + /** + * Redact a piece of text through the configured profile. + */ + private function redact(string $text): string + { + $out = $this->redactor->redactSafely($text, $this->profile); + + return is_string($out) ? $out : (string) json_encode($out); + } +} diff --git a/src/Support/AllowList.php b/src/Support/AllowList.php new file mode 100644 index 0000000..ab4deae --- /dev/null +++ b/src/Support/AllowList.php @@ -0,0 +1,128 @@ + */ + private static array $memo = []; + + /** @var array */ + private array $exact = []; + + /** @var array */ + private array $patterns = []; + + /** + * Create a new allow list instance. + * + * @param array $entries + */ + private function __construct(array $entries) + { + foreach ($entries as $entry) { + $entry = trim($entry); + + if ($entry === '') { + continue; + } + + if ($this->looksLikeRegex($entry) && Pcre::isValidPattern($entry)) { + $this->patterns[] = $entry; + + continue; + } + + $this->exact[mb_strtolower($entry)] = true; + } + } + + /** + * Compile an entry list, reusing the result for identical lists. + * + * @param array $entries + */ + public static function for(array $entries): self + { + $cacheKey = implode("\0", $entries); + + return self::$memo[$cacheKey] ??= new self($entries); + } + + /** + * Get an empty allow list. + */ + public static function none(): self + { + return self::for([]); + } + + /** + * Determine if the list has no entries. + */ + public function isEmpty(): bool + { + return $this->exact === [] && $this->patterns === []; + } + + /** + * Determine if the value is allowed. + */ + public function allows(string $value): bool + { + if ($this->isEmpty()) { + return false; + } + + if (isset($this->exact[mb_strtolower(trim($value))])) { + return true; + } + + foreach ($this->patterns as $pattern) { + // onError: false, since an entry that cannot be evaluated excuses nothing... + if (Pcre::matches($pattern, $value, onError: false, rule: 'allowlist')) { + return true; + } + } + + return false; + } + + /** + * Determine if an entry has a leading delimiter that closes before an optional modifier suffix. + */ + private function looksLikeRegex(string $entry): bool + { + if (strlen($entry) < 3) { + return false; + } + + $delimiter = $entry[0]; + + if (ctype_alnum($delimiter) || $delimiter === '\\' || ctype_space($delimiter)) { + return false; + } + + $close = match ($delimiter) { + '(' => ')', + '[' => ']', + '{' => '}', + '<' => '>', + default => $delimiter, + }; + + return preg_match('/'.preg_quote($close, '/').'[imsxuADSUXJn]*$/', substr($entry, 1)) === 1; + } +} diff --git a/src/Support/Configuration.php b/src/Support/Configuration.php new file mode 100644 index 0000000..6e82a6e --- /dev/null +++ b/src/Support/Configuration.php @@ -0,0 +1,49 @@ +get($key, $default); + } + + /** + * Get the configuration repository of the current container. + */ + public static function repository(): Repository + { + $container = Container::getInstance(); + + if (! self::$repository instanceof Repository || self::$container !== $container) { + /** @var Repository $repository */ + $repository = $container->make('config'); + + self::$container = $container; + self::$repository = $repository; + } + + return self::$repository; + } +} diff --git a/src/Support/DeterministicRandom.php b/src/Support/DeterministicRandom.php new file mode 100644 index 0000000..fc6805d --- /dev/null +++ b/src/Support/DeterministicRandom.php @@ -0,0 +1,108 @@ +buffer === '') { + $this->buffer = hash_hmac('sha256', $this->seed.'|'.$this->counter++, $this->key, true); + } + + $byte = ord($this->buffer[0]); + $this->buffer = substr($this->buffer, 1); + + return $byte; + } + + /** + * Get a value in [0, $bound). + * + * Uses rejection sampling so the distribution is not skewed by a modulo fold. + */ + public function below(int $bound): int + { + if ($bound <= 1) { + return 0; + } + + // Draw enough bytes to cover the range, then reject anything landing in the partial final window... + $bytes = (int) ceil(log(max($bound, 2), 256)); + $max = 256 ** $bytes; + $limit = $max - ($max % $bound); + + $value = 0; + + // Sixty-four rejections in a row is astronomically unlikely, so the last draw is folded rather than looping forever... + for ($attempt = 0; $attempt < 64; $attempt++) { + $value = 0; + for ($i = 0; $i < $bytes; $i++) { + $value = ($value << 8) | $this->byte(); + } + + if ($value < $limit) { + break; + } + } + + return $value % $bound; + } + + /** + * Pick one character from the alphabet. + */ + public function pick(string $alphabet): string + { + $length = strlen($alphabet); + + return $length === 0 ? '' : $alphabet[$this->below($length)]; + } + + /** + * Get a decimal digit. + */ + public function digit(): string + { + return (string) $this->below(10); + } + + /** + * Generate a token of the given length from the alphabet. + */ + public function token(int $length, string $alphabet = 'abcdefghijkmnopqrstuvwxyz23456789'): string + { + $out = ''; + for ($i = 0; $i < $length; $i++) { + $out .= $this->pick($alphabet); + } + + return $out; + } +} diff --git a/src/Support/InternalLog.php b/src/Support/InternalLog.php new file mode 100644 index 0000000..5ba87d7 --- /dev/null +++ b/src/Support/InternalLog.php @@ -0,0 +1,52 @@ + $context + */ + public static function warning(string $message, array $context = []): void + { + if (self::$emitting) { + return; + } + + self::$emitting = true; + + try { + Log::warning($message, $context); + } catch (Throwable) { + // A broken logger must not turn into a broken application... + } finally { + self::$emitting = false; + } + } + + /** + * Determine if a diagnostic is currently being emitted. + */ + public static function isEmitting(): bool + { + return self::$emitting; + } +} diff --git a/src/Support/KeyMatcher.php b/src/Support/KeyMatcher.php new file mode 100644 index 0000000..c5f6783 --- /dev/null +++ b/src/Support/KeyMatcher.php @@ -0,0 +1,182 @@ + */ + private static array $memo = []; + + /** @var array */ + private array $exact = []; + + /** @var array */ + private array $contains = []; + + /** @var array */ + private array $prefix = []; + + /** @var array */ + private array $suffix = []; + + /** @var array */ + private array $regex = []; + + private bool $matchesEverything = false; + + private bool $empty = true; + + /** + * Create a new key matcher instance. + * + * @param array $patterns + */ + private function __construct(array $patterns) + { + foreach ($patterns as $pattern) { + $this->compile(strtolower($pattern)); + } + } + + /** + * Compile a pattern list, reusing the result for identical lists. + * + * @param array $patterns + */ + public static function for(array $patterns): self + { + $cacheKey = implode("\0", $patterns); + + return self::$memo[$cacheKey] ??= new self($patterns); + } + + /** + * Flush the compiled matcher cache. + */ + public static function flush(): void + { + self::$memo = []; + } + + /** + * Determine if the matcher has no patterns. + */ + public function isEmpty(): bool + { + return $this->empty; + } + + /** + * Determine if the key matches any compiled pattern. + * + * @param bool $onError what a PCRE failure should be reported as + */ + public function matches(string $key, bool $onError = true): bool + { + if ($this->empty || $key === '') { + return false; + } + + if ($this->matchesEverything) { + return true; + } + + $key = strtolower($key); + + if (isset($this->exact[$key])) { + return true; + } + + foreach ($this->contains as $needle) { + if (str_contains($key, $needle)) { + return true; + } + } + + foreach ($this->prefix as $needle) { + if (str_starts_with($key, $needle)) { + return true; + } + } + + foreach ($this->suffix as $needle) { + if (str_ends_with($key, $needle)) { + return true; + } + } + + foreach ($this->regex as $compiled) { + if (Pcre::matches($compiled['pattern'], $key, $onError, 'key_pattern:'.$compiled['source'])) { + return true; + } + } + + return false; + } + + /** + * Compile one pattern into the cheapest test for its shape. + */ + private function compile(string $pattern): void + { + if ($pattern === '') { + return; + } + + $this->empty = false; + + if (! str_contains($pattern, '*')) { + $this->exact[$pattern] = true; + + return; + } + + if (trim($pattern, '*') === '') { + // '*', '**' and so on match everything... + $this->matchesEverything = true; + + return; + } + + $core = trim($pattern, '*'); + + // Only the outer wildcards are special-cased, since an interior '*' needs real backtracking and goes to PCRE... + if (! str_contains($core, '*')) { + $leading = str_starts_with($pattern, '*'); + $trailing = str_ends_with($pattern, '*'); + + if ($leading && $trailing) { + $this->contains[] = $core; + + return; + } + + if ($trailing) { + $this->prefix[] = $core; + + return; + } + + $this->suffix[] = $core; + + return; + } + + $this->regex[] = [ + 'pattern' => '/^'.str_replace('\*', '.*', preg_quote($pattern, '/')).'$/i', + 'source' => $pattern, + ]; + } +} diff --git a/src/Support/Pcre.php b/src/Support/Pcre.php new file mode 100644 index 0000000..a994fe8 --- /dev/null +++ b/src/Support/Pcre.php @@ -0,0 +1,83 @@ +): string $callback + */ + public static function replaceCallback( + string $pattern, + callable $callback, + string $subject, + ?string $rule = null, + int $flags = 0 + ): ?string { + $result = @preg_replace_callback($pattern, $callback, $subject, -1, $count, $flags); + + if ($result === null || preg_last_error() !== PREG_NO_ERROR) { + self::reportFailure($pattern, $rule, strlen($subject)); + + return null; + } + + return $result; + } + + /** + * Determine if a pattern compiles at all. + */ + public static function isValidPattern(string $pattern): bool + { + return @preg_match($pattern, '') !== false; + } + + /** + * Log an engine failure. + */ + private static function reportFailure(string $pattern, ?string $rule, int $subjectLength): void + { + InternalLog::warning('Redaction pattern failed to evaluate; failing closed', [ + 'rule' => $rule, + 'pattern' => $pattern, + 'subject_length' => $subjectLength, + 'preg_error' => preg_last_error_msg(), + ]); + } +} diff --git a/src/Support/Pseudonymizer.php b/src/Support/Pseudonymizer.php new file mode 100644 index 0000000..7213eb5 --- /dev/null +++ b/src/Support/Pseudonymizer.php @@ -0,0 +1,106 @@ +key, $this->seed($entity, $value)); + } + + /** + * Get a short, stable, URL-safe identifier for a value. + */ + public function token(string $entity, string $value, int $length = 10): string + { + return $this->random($entity, $value)->token($length); + } + + /** + * Get a full hex digest for a value, for correlating without any pretence of the original's shape. + */ + public function digest(string $entity, string $value): string + { + return hash_hmac('sha256', $this->seed($entity, $value), $this->key); + } + + /** + * Get the normalised seed for a value. + * + * "Bob@Example.COM " and "bob@example.com" are the same person, and a + * mapping that disagrees is not joinable. + */ + private function seed(string $entity, string $value): string + { + return $this->salt.'|'.$entity.'|'.mb_strtolower(trim($value)); + } +} diff --git a/src/Support/SecretRegistry.php b/src/Support/SecretRegistry.php new file mode 100644 index 0000000..5d47515 --- /dev/null +++ b/src/Support/SecretRegistry.php @@ -0,0 +1,93 @@ + value => entity */ + private array $secrets = []; + + /** Length of the shortest registered value; a shorter subject cannot contain one. */ + private int $shortest = PHP_INT_MAX; + + /** + * Register one value, returning false if it was too short to be safe. + */ + public function add(string $value, string $entity = 'known_secret'): bool + { + if (strlen($value) < self::MIN_LENGTH) { + return false; + } + + $this->secrets[$value] = $entity; + $this->shortest = min($this->shortest, strlen($value)); + + return true; + } + + /** + * Determine if a subject is long enough to contain any registered value. + */ + public function couldContainOne(string $subject): bool + { + return $this->secrets !== [] && strlen($subject) >= $this->shortest; + } + + /** + * Get the number of registered values. + */ + public function count(): int + { + return count($this->secrets); + } + + /** + * Get every occurrence of every registered value in the subject. + * + * @return array + */ + public function find(string $subject): array + { + $found = []; + + foreach ($this->secrets as $secret => $entity) { + $offset = 0; + + while (($position = strpos($subject, $secret, $offset)) !== false) { + $found[] = ['offset' => $position, 'value' => $secret, 'entity' => $entity]; + $offset = $position + strlen($secret); + } + } + + return $found; + } + + /** + * Merge another registry's values into a copy of this one. + */ + public function merge(self $other): self + { + $merged = clone $this; + + foreach ($other->secrets as $value => $entity) { + $merged->secrets[$value] = $entity; + $merged->shortest = min($merged->shortest, strlen($value)); + } + + return $merged; + } +} diff --git a/src/Testing/RedactorFake.php b/src/Testing/RedactorFake.php new file mode 100644 index 0000000..d2a1d48 --- /dev/null +++ b/src/Testing/RedactorFake.php @@ -0,0 +1,190 @@ + + */ + protected array $calls = []; + + public function inspect(mixed $content, ?string $profile = null, ?bool $mark = null, ?EntityFilter $entities = null): RedactionResult + { + $result = parent::inspect($content, $profile, $mark, $entities); + + $this->calls[] = ['profile' => $profile, 'input' => $content, 'result' => $result]; + + return $result; + } + + /** + * Get every recorded call, oldest first. + * + * @return array + */ + public function recorded(): array + { + return $this->calls; + } + + public function forget(): void + { + $this->calls = []; + } + + /** + * Assert that none of the given secrets appeared in anything the redactor produced. + * + * The strongest thing a test can say about redaction: not "this key was + * handled" but "this secret did not get out", across every call. + */ + public function assertNeverEmitted(string ...$secrets): void + { + Assert::assertNotEmpty( + $this->calls, + 'No redaction calls were recorded. Was the fake installed before the code under test resolved the redactor?' + ); + + foreach ($this->calls as $index => $call) { + $output = $this->stringify($call['result']->value); + + foreach ($secrets as $secret) { + Assert::assertStringNotContainsString( + $secret, + $output, + sprintf('Call #%d (profile %s) emitted a value that should have been redacted.', $index + 1, $call['profile'] ?? 'default') + ); + } + } + } + + /** + * Assert that at least one call redacted something under the given key. + */ + public function assertRedacted(string $key): void + { + Assert::assertTrue( + $this->anyCall(fn (RedactionResult $r): bool => in_array($key, $r->redactedKeys, true)), + sprintf('No redaction recorded under key [%s]. Keys redacted: %s.', $key, $this->describeKeys()) + ); + } + + public function assertNotRedacted(string $key): void + { + Assert::assertFalse( + $this->anyCall(fn (RedactionResult $r): bool => in_array($key, $r->redactedKeys, true)), + sprintf('A redaction was recorded under key [%s], which should have been left alone.', $key) + ); + } + + /** + * Assert that at least one call produced a finding from the given rule. + */ + public function assertFinding(string $rule): void + { + Assert::assertTrue( + $this->anyCall(function (RedactionResult $r) use ($rule): bool { + foreach ($r->findings as $finding) { + if ($finding->rule === $rule) { + return true; + } + } + + return false; + }), + sprintf('No finding from rule [%s] was recorded.', $rule) + ); + } + + public function assertSomethingRedacted(): void + { + Assert::assertTrue( + $this->anyCall(fn (RedactionResult $r): bool => $r->wasRedacted), + 'Nothing was redacted in any call.' + ); + } + + public function assertNothingRedacted(): void + { + Assert::assertFalse( + $this->anyCall(fn (RedactionResult $r): bool => $r->wasRedacted), + sprintf('Something was redacted. Keys: %s.', $this->describeKeys()) + ); + } + + public function assertProfileUsed(string $profile): void + { + $used = array_values(array_unique(array_map(fn (array $c) => $c['profile'] ?? 'default', $this->calls))); + + Assert::assertContains( + $profile, + $used, + sprintf('Profile [%s] was never used. Profiles used: %s.', $profile, $used === [] ? 'none' : implode(', ', $used)) + ); + } + + public function assertCalled(int $times): void + { + Assert::assertCount($times, $this->calls, sprintf('Expected %d redaction calls, recorded %d.', $times, count($this->calls))); + } + + public function assertNotCalled(): void + { + $this->assertCalled(0); + } + + /** + * @param callable(RedactionResult): bool $predicate + */ + private function anyCall(callable $predicate): bool + { + foreach ($this->calls as $call) { + if ($predicate($call['result'])) { + return true; + } + } + + return false; + } + + private function describeKeys(): string + { + $keys = []; + + foreach ($this->calls as $call) { + $keys = [...$keys, ...$call['result']->redactedKeys]; + } + + $keys = array_values(array_unique($keys)); + + return $keys === [] ? 'none' : implode(', ', $keys); + } + + private function stringify(mixed $value): string + { + if (is_string($value)) { + return $value; + } + + $encoded = json_encode($value, JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE | JSON_PARTIAL_OUTPUT_ON_ERROR); + + return $encoded === false ? '' : $encoded; + } +} diff --git a/src/Tokenization/CacheTokenStore.php b/src/Tokenization/CacheTokenStore.php new file mode 100644 index 0000000..167bec6 --- /dev/null +++ b/src/Tokenization/CacheTokenStore.php @@ -0,0 +1,77 @@ +encrypter->encryptString($entity."\0".$value); + $ttl = $ttlSeconds ?? $this->defaultTtlSeconds; + + if ($ttl === null) { + $this->cache->forever($this->prefix.$token, $payload); + } else { + $this->cache->put($this->prefix.$token, $payload, $ttl); + } + } + + /** + * Get the original value for a token, if it is known. + */ + public function get(string $token): ?string + { + $payload = $this->cache->get($this->prefix.$token); + + if (! is_string($payload)) { + return null; + } + + try { + $decrypted = $this->encrypter->decryptString($payload); + } catch (Throwable) { + // A key rotation or a corrupt entry means the token is simply unknown... + return null; + } + + $separator = strpos($decrypted, "\0"); + + return $separator === false ? $decrypted : substr($decrypted, $separator + 1); + } + + /** + * Forget a token. + */ + public function forget(string $token): void + { + $this->cache->forget($this->prefix.$token); + } +} diff --git a/src/Tokenization/Detokenizer.php b/src/Tokenization/Detokenizer.php new file mode 100644 index 0000000..f129e25 --- /dev/null +++ b/src/Tokenization/Detokenizer.php @@ -0,0 +1,79 @@ +replaceIn($content); + } + + if (is_array($content)) { + $out = []; + + foreach ($content as $key => $value) { + $out[$key] = $this->detokenize($value); + } + + return $out; + } + + return $content; + } + + /** + * Get every token in a string, whether or not the store knows it. + * + * @return array + */ + public function tokensIn(string $text): array + { + preg_match_all($this->pattern(), $text, $matches); + + return array_values(array_unique($matches[0])); + } + + /** + * Replace every known token in a string. + */ + private function replaceIn(string $text): string + { + if (! str_contains($text, $this->prefix.'_')) { + return $text; + } + + $result = preg_replace_callback($this->pattern(), fn (array $m): string => $this->store->get($m[0]) ?? $m[0], $text); + + return $result ?? $text; + } + + /** + * Get the pattern that matches a token. + */ + private function pattern(): string + { + return '/\b'.preg_quote($this->prefix, '/').'_[a-z0-9]+(?:_[a-z0-9]+)*_[a-z0-9]{'.TokenizeOperator::ID_LENGTH.'}\b/'; + } +} diff --git a/src/Tokenization/LazyTokenStore.php b/src/Tokenization/LazyTokenStore.php new file mode 100644 index 0000000..8d88ed4 --- /dev/null +++ b/src/Tokenization/LazyTokenStore.php @@ -0,0 +1,60 @@ +store()->put($token, $value, $entity, $ttlSeconds); + } + + /** + * Get the original value for a token, if it is known. + */ + public function get(string $token): ?string + { + return $this->store()->get($token); + } + + /** + * Forget a token. + */ + public function forget(string $token): void + { + $this->store()->forget($token); + } + + /** + * Resolve the underlying store. + */ + private function store(): TokenStore + { + return $this->resolved ??= ($this->resolver)(); + } +} diff --git a/src/Tokenization/TokenStore.php b/src/Tokenization/TokenStore.php new file mode 100644 index 0000000..1dd56cc --- /dev/null +++ b/src/Tokenization/TokenStore.php @@ -0,0 +1,32 @@ + tok_email_k4m9rp2xzq + * + * The token is derived the way a surrogate is, keyed and stable, so it stays + * joinable, and it is spelt to survive a language model: one word, no + * punctuation a tokenizer would split on, an entity name a model can reason + * about. The original goes into the token store, encrypted, for as long as + * the store's TTL allows. Without a pseudonymization key there is no stable + * token to make, so the span is redacted instead. + */ +class TokenizeOperator implements Operator +{ + public const PREFIX = 'tok'; + + public const ID_LENGTH = 12; + + /** + * Create a new tokenize operator instance. + */ + public function __construct( + private readonly TokenStore $store, + private readonly string $prefix = self::PREFIX, + ) {} + + /** + * Replace the span with a token and store the original. + */ + public function apply(Detection $detection, OperatorContext $context): string + { + $pseudonymizer = $context->pseudonymizer(); + + if (! $pseudonymizer instanceof Pseudonymizer) { + return $context->replacement; + } + + $entity = preg_replace('/[^a-z0-9]+/', '_', strtolower($detection->entity)) ?? 'value'; + $entity = trim($entity, '_') ?: 'value'; + + $token = sprintf('%s_%s_%s', $this->prefix, $entity, $pseudonymizer->token('tokenize:'.$detection->entity, $detection->value, self::ID_LENGTH)); + + $ttl = $context->intOption('ttl', -1); + + $this->store->put($token, $detection->value, $detection->entity, $ttl < 0 ? null : $ttl); + + return $token; + } +} diff --git a/src/Verification/SecretVerifier.php b/src/Verification/SecretVerifier.php new file mode 100644 index 0000000..ad9d3f4 --- /dev/null +++ b/src/Verification/SecretVerifier.php @@ -0,0 +1,145 @@ + */ + private readonly array $verifiers; + + /** + * @param array $allowed verifier names permitted to run + * @param array|null $verifiers overridable for testing + */ + public function __construct( + private readonly array $allowed = [], + ?array $verifiers = null, + ) { + $this->verifiers = $verifiers ?? [ + new GitHubTokenVerifier, + new StripeKeyVerifier, + new SlackTokenVerifier, + new OpenAiKeyVerifier, + new AnthropicKeyVerifier, + new SendGridKeyVerifier, + new GoogleApiKeyVerifier, + ...static::$registered, + ]; + } + + /** @var array */ + protected static array $registered = []; + + /** + * Register a verifier of your own. It still has to be allow-listed to run. + */ + public static function register(Verifier $verifier): void + { + static::$registered[] = $verifier; + } + + /** + * Create a verifier from config, or null if config does not permit any. + * + * @param array $settings + * @param array|null $verifiers + */ + public static function fromConfig(array $settings, ?array $verifiers = null): ?self + { + if (($settings['enabled'] ?? false) !== true) { + return null; + } + + $allowed = $settings['verifiers'] ?? []; + $allowed = is_array($allowed) ? array_values(array_filter($allowed, is_string(...))) : []; + + // An empty allowlist means "none", not "all"; enabling is separate from choosing who to trust... + return $allowed === [] ? null : new self($allowed, $verifiers); + } + + /** + * Get the verifiers that are permitted to run. + * + * @return array + */ + public function enabled(): array + { + return array_values(array_filter( + $this->verifiers, + fn (Verifier $v): bool => in_array($v->name(), $this->allowed, true) + )); + } + + /** + * Get every host a run could contact, so the operator can be told up front. + * + * @return array + */ + public function hosts(): array + { + $hosts = array_map(fn (Verifier $v): string => $v->host(), $this->enabled()); + sort($hosts); + + return array_values(array_unique($hosts)); + } + + public function canVerify(string $entity, string $rule): bool + { + return $this->verifierFor($entity, $rule) instanceof Verifier; + } + + /** + * Verify one secret, or report Unknown if nothing is allowed to. + * + * Never throws: a verification failure must degrade the finding to Unknown, + * not abandon a scan that has already found real problems. + */ + public function verify(string $entity, string $rule, string $secret): VerificationResult + { + $verifier = $this->verifierFor($entity, $rule); + + if (! $verifier instanceof Verifier) { + return VerificationResult::unknown('No verifier is enabled for this kind of credential.'); + } + + try { + return $verifier->verify($secret)->withVerifier($verifier->name()); + } catch (Throwable $e) { + return VerificationResult::unknown( + 'The verifier failed: '.$e->getMessage(), + $verifier->name() + ); + } + } + + private function verifierFor(string $entity, string $rule): ?Verifier + { + foreach ($this->enabled() as $verifier) { + if ($verifier->supports($entity, $rule)) { + return $verifier; + } + } + + return null; + } +} diff --git a/src/Verification/VerificationResult.php b/src/Verification/VerificationResult.php new file mode 100644 index 0000000..3aaef5a --- /dev/null +++ b/src/Verification/VerificationResult.php @@ -0,0 +1,54 @@ +status, $this->note, $verifier); + } + + /** + * @return array + */ + public function toArray(): array + { + return array_filter([ + 'status' => $this->status->value, + 'note' => $this->note, + 'verifier' => $this->verifier, + ], fn (?string $v): bool => $v !== null); + } +} diff --git a/src/Verification/VerificationStatus.php b/src/Verification/VerificationStatus.php new file mode 100644 index 0000000..80f3543 --- /dev/null +++ b/src/Verification/VerificationStatus.php @@ -0,0 +1,46 @@ + 'critical', + self::Unknown => 'high', + self::Inactive => 'low', + }; + } +} diff --git a/src/Verification/Verifier.php b/src/Verification/Verifier.php new file mode 100644 index 0000000..e20e543 --- /dev/null +++ b/src/Verification/Verifier.php @@ -0,0 +1,37 @@ + $secret, + 'anthropic-version' => '2023-06-01', + ])->timeout(5)->get('https://api.anthropic.com/v1/models'); + + if ($response->status() === 401) { + return VerificationResult::inactive('Anthropic rejected the key (401).'); + } + + if ($response->successful()) { + return VerificationResult::active('Anthropic accepted the key; it is live and should be revoked.'); + } + + return VerificationResult::unknown(sprintf('Anthropic returned %d.', $response->status())); + } catch (Throwable $e) { + return VerificationResult::unknown('Could not reach Anthropic: '.$e->getMessage()); + } + } +} diff --git a/src/Verification/Verifiers/GitHubTokenVerifier.php b/src/Verification/Verifiers/GitHubTokenVerifier.php new file mode 100644 index 0000000..3425d8a --- /dev/null +++ b/src/Verification/Verifiers/GitHubTokenVerifier.php @@ -0,0 +1,59 @@ + 'Bearer '.$secret, + 'Accept' => 'application/vnd.github+json', + 'User-Agent' => 'kirschbaum-redactor', + ])->timeout(5)->get('https://api.github.com/user'); + + if ($response->status() === 401) { + return VerificationResult::inactive('GitHub rejected the token (401).'); + } + + if ($response->successful()) { + return VerificationResult::active('GitHub accepted the token; it is live and should be revoked.'); + } + + return VerificationResult::unknown(sprintf('GitHub returned %d.', $response->status())); + } catch (Throwable $e) { + // The message is safe to surface; the secret never appears in it... + return VerificationResult::unknown('Could not reach GitHub: '.$e->getMessage()); + } + } +} diff --git a/src/Verification/Verifiers/GoogleApiKeyVerifier.php b/src/Verification/Verifiers/GoogleApiKeyVerifier.php new file mode 100644 index 0000000..e22660d --- /dev/null +++ b/src/Verification/Verifiers/GoogleApiKeyVerifier.php @@ -0,0 +1,55 @@ +get('https://generativelanguage.googleapis.com/v1/models', ['key' => $secret]); + + if ($response->status() === 400) { + return VerificationResult::inactive('Google rejected the key (400, API key not valid).'); + } + + if ($response->successful() || $response->status() === 403) { + return VerificationResult::active('Google recognised the key; it is live and should be revoked.'); + } + + return VerificationResult::unknown(sprintf('Google returned %d.', $response->status())); + } catch (Throwable $e) { + return VerificationResult::unknown('Could not reach Google: '.$e->getMessage()); + } + } +} diff --git a/src/Verification/Verifiers/OpenAiKeyVerifier.php b/src/Verification/Verifiers/OpenAiKeyVerifier.php new file mode 100644 index 0000000..36251a2 --- /dev/null +++ b/src/Verification/Verifiers/OpenAiKeyVerifier.php @@ -0,0 +1,53 @@ +timeout(5) + ->get('https://api.openai.com/v1/models'); + + if ($response->status() === 401) { + return VerificationResult::inactive('OpenAI rejected the key (401).'); + } + + if ($response->successful()) { + return VerificationResult::active('OpenAI accepted the key; it is live and should be revoked.'); + } + + return VerificationResult::unknown(sprintf('OpenAI returned %d.', $response->status())); + } catch (Throwable $e) { + return VerificationResult::unknown('Could not reach OpenAI: '.$e->getMessage()); + } + } +} diff --git a/src/Verification/Verifiers/SendGridKeyVerifier.php b/src/Verification/Verifiers/SendGridKeyVerifier.php new file mode 100644 index 0000000..b854e80 --- /dev/null +++ b/src/Verification/Verifiers/SendGridKeyVerifier.php @@ -0,0 +1,53 @@ +timeout(5) + ->get('https://api.sendgrid.com/v3/scopes'); + + if (in_array($response->status(), [401, 403], true)) { + return VerificationResult::inactive(sprintf('SendGrid rejected the key (%d).', $response->status())); + } + + if ($response->successful()) { + return VerificationResult::active('SendGrid accepted the key; it is live and should be revoked.'); + } + + return VerificationResult::unknown(sprintf('SendGrid returned %d.', $response->status())); + } catch (Throwable $e) { + return VerificationResult::unknown('Could not reach SendGrid: '.$e->getMessage()); + } + } +} diff --git a/src/Verification/Verifiers/SlackTokenVerifier.php b/src/Verification/Verifiers/SlackTokenVerifier.php new file mode 100644 index 0000000..39a73a4 --- /dev/null +++ b/src/Verification/Verifiers/SlackTokenVerifier.php @@ -0,0 +1,67 @@ +timeout(5) + ->post('https://slack.com/api/auth.test'); + + if (! $response->successful()) { + return VerificationResult::unknown(sprintf('Slack returned %d.', $response->status())); + } + + $ok = $response->json('ok'); + + if ($ok === true) { + return VerificationResult::active('Slack accepted the token; it is live and should be revoked.'); + } + + if ($ok === false) { + $error = $response->json('error'); + + return VerificationResult::inactive(sprintf( + 'Slack rejected the token (%s).', + is_string($error) ? $error : 'not ok' + )); + } + + return VerificationResult::unknown('Slack returned an unexpected response.'); + } catch (Throwable $e) { + return VerificationResult::unknown('Could not reach Slack: '.$e->getMessage()); + } + } +} diff --git a/src/Verification/Verifiers/StripeKeyVerifier.php b/src/Verification/Verifiers/StripeKeyVerifier.php new file mode 100644 index 0000000..6012cc4 --- /dev/null +++ b/src/Verification/Verifiers/StripeKeyVerifier.php @@ -0,0 +1,56 @@ +timeout(5) + ->get('https://api.stripe.com/v1/balance'); + + if ($response->status() === 401) { + return VerificationResult::inactive('Stripe rejected the key (401).'); + } + + if ($response->successful()) { + return VerificationResult::active('Stripe accepted the key; it is live and should be rolled.'); + } + + return VerificationResult::unknown(sprintf('Stripe returned %d.', $response->status())); + } catch (Throwable $e) { + return VerificationResult::unknown('Could not reach Stripe: '.$e->getMessage()); + } + } +} diff --git a/stubs/pre-commit b/stubs/pre-commit new file mode 100755 index 0000000..a47e049 --- /dev/null +++ b/stubs/pre-commit @@ -0,0 +1,9 @@ +#!/bin/sh +# +# Fails the commit when a staged change adds a secret. +# +# Scans only the lines being committed, so pre-existing findings never block +# a commit - accept those into the baseline or mark them `redactor:allow`. +# Install with: git config core.hooksPath .githooks (or copy into .git/hooks) + +php artisan redactor:scan --staged --bail diff --git a/stubs/redactor-scan.yml b/stubs/redactor-scan.yml new file mode 100644 index 0000000..365bede --- /dev/null +++ b/stubs/redactor-scan.yml @@ -0,0 +1,43 @@ +name: Secret scan + +on: + pull_request: + push: + branches: [main] + +jobs: + redactor: + name: redactor:scan + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + security-events: write + steps: + - uses: actions/checkout@v4 + with: + # The diff and history modes need more than the tip commit. + fetch-depth: 0 + + - uses: shivammathur/setup-php@v2 + with: + php-version: '8.4' + coverage: none + + - run: composer install --no-interaction --prefer-dist --no-progress + + # On a pull request: fail on secrets the branch adds over its base. + - name: Scan the changes in this pull request + if: github.event_name == 'pull_request' + run: php artisan redactor:scan --diff=origin/${{ github.base_ref }} --bail --output=sarif > redactor.sarif + + # On main: the whole tree, against the committed baseline. + - name: Scan the repository + if: github.event_name != 'pull_request' + run: php artisan redactor:scan --bail --output=sarif > redactor.sarif + + - name: Upload findings to code scanning + if: always() + uses: github/codeql-action/upload-sarif@v3 + with: + sarif_file: redactor.sarif diff --git a/tests/Feature/AiRedactPromptTest.php b/tests/Feature/AiRedactPromptTest.php new file mode 100644 index 0000000..de472be --- /dev/null +++ b/tests/Feature/AiRedactPromptTest.php @@ -0,0 +1,89 @@ +set('app.key', 'base64:'.base64_encode(random_bytes(32))); + config()->set('redactor.pseudonymization.key', testPseudonymizationKey()); + config()->set('redactor.profiles.ai', [ + 'enabled' => true, + 'strategies' => [RegexPatternsStrategy::class], + 'safe_keys' => [], + 'blocked_keys' => [], + 'patterns' => ['email' => ['pattern' => '/[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+/', 'entity' => 'email']], + 'operators' => ['default' => 'tokenize'], + 'replacement' => '[REDACTED]', + 'mark_redacted' => false, + 'track_redacted_keys' => false, + 'non_redactable_object_behavior' => 'preserve', + 'max_value_length' => null, + 'redact_large_objects' => false, + 'max_object_size' => 100, + 'shannon_entropy' => ['enabled' => false], + ]); + }); + + it('redacts the prompt the provider sees and resolves tokens in the answer', function (): void { + $seen = null; + + $response = RedactPrompt::using('ai')->handle(agentPrompt('Reply to alice@customer.com politely'), function (AgentPrompt $prompt) use (&$seen): AgentResponse { + $seen = $prompt->prompt; + + preg_match('/tok_email_[a-z0-9]{12}/', $prompt->prompt, $m); + + return agentResponse("Dear {$m[0]}, thank you."); + }); + + expect($seen)->toMatch('/^Reply to tok_email_[a-z0-9]{12} politely$/') + ->and($seen)->not->toContain('alice@customer.com') + ->and($response->text)->toBe('Dear alice@customer.com, thank you.'); + }); + + it('redacts outright with a profile that does not tokenise, and touches nothing on the way back', function (): void { + $middleware = new RedactPrompt(resolve(Redactor::class), 'default'); + + $response = $middleware->handle(agentPrompt('Reply to alice@customer.com'), function (AgentPrompt $prompt): AgentResponse { + expect($prompt->prompt)->toBe('Reply to [REDACTED]'); + + return agentResponse('Done.'); + }); + + expect($response->text)->toBe('Done.'); + }); + + it('can leave tokens in the answer when asked', function (): void { + $response = RedactPrompt::using('ai', detokenizeResponse: false)->handle(agentPrompt('alice@customer.com'), fn (AgentPrompt $prompt): AgentResponse => agentResponse('echo '.$prompt->prompt)); + + expect($response->text)->toMatch('/^echo tok_email_[a-z0-9]{12}$/'); + }); +}); diff --git a/tests/Feature/McpRedactsResponsesTest.php b/tests/Feature/McpRedactsResponsesTest.php new file mode 100644 index 0000000..5943259 --- /dev/null +++ b/tests/Feature/McpRedactsResponsesTest.php @@ -0,0 +1,241 @@ + 7, 'email' => 'bob@example.com', 'password' => 'hunter2']); + } +} + +class JsonTextTool extends Tool +{ + protected string $description = 'Returns JSON as text'; + + public function handle(Request $request): Response + { + return Response::json(['id' => 7, 'email' => 'bob@example.com']); + } +} + +class BlobTool extends Tool +{ + protected string $description = 'Returns an image'; + + public function handle(Request $request): Response + { + return Response::image(base64_encode(random_bytes(400)), 'image/png'); + } +} + +class ErrorTool extends Tool +{ + protected string $description = 'Fails'; + + public function handle(Request $request): Response + { + return Response::error('Could not reach postgres://app:s3cr3t@db.internal/app'); + } +} + +class CustomerResource extends Resource +{ + protected string $description = 'A customer file'; + + public function handle(Request $request): Response + { + return Response::text("name: Bob\nemail: bob@example.com\n"); + } +} + +class SummaryPrompt extends Prompt +{ + protected string $description = 'Summarise a ticket'; + + public function handle(Request $request): Response + { + return Response::text('Summarise the ticket from bob@example.com about card 4111111111111111'); + } +} + +class RedactedServer extends Server +{ + use RedactsResponses; + + protected array $tools = [LeakyTool::class, StructuredTool::class, JsonTextTool::class, BlobTool::class, ErrorTool::class]; + + protected array $resources = [CustomerResource::class]; + + protected array $prompts = [SummaryPrompt::class]; +} + +class ObservabilityServer extends RedactedServer +{ + protected function redactionProfile(): ?string + { + return 'observability'; + } +} + +describe('RedactsResponses on an MCP server', function (): void { + it('redacts a tool\'s text content', function (): void { + RedactedServer::tool(LeakyTool::class) + ->assertOk() + ->assertDontSee('bob@example.com') + ->assertDontSee('sk_live_4eC39HqLyjWDarjtT1zdp7dc') + ->assertSee('[REDACTED]') + ->assertSee('************1111'); + }); + + it('redacts structured content as data, keeping its shape', function (): void { + RedactedServer::tool(StructuredTool::class) + ->assertOk() + ->assertStructuredContent(['id' => 7, 'email' => '[REDACTED]', 'password' => '[REDACTED]']); + }); + + it('redacts JSON returned as text', function (): void { + RedactedServer::tool(JsonTextTool::class) + ->assertOk() + ->assertDontSee('bob@example.com') + ->assertSee('"id":7'); + }); + + it('leaves binary content alone', function (): void { + $response = RedactedServer::tool(BlobTool::class)->assertOk(); + + $response->assertDontSee('[REDACTED]'); + }); + + it('redacts an error message', function (): void { + RedactedServer::tool(ErrorTool::class) + ->assertHasErrors() + ->assertDontSee('s3cr3t') + ->assertSee('postgres://app:[REDACTED]@db.internal/app'); + }); + + it('redacts a resource read', function (): void { + RedactedServer::resource(CustomerResource::class) + ->assertOk() + ->assertDontSee('bob@example.com') + ->assertSee('email: [REDACTED]'); + }); + + it('redacts a prompt\'s messages', function (): void { + RedactedServer::prompt(SummaryPrompt::class) + ->assertOk() + ->assertDontSee('bob@example.com') + ->assertSee('************1111'); + }); + + it('honours the profile the server names', function (): void { + config()->set('app.key', 'base64:'.base64_encode(random_bytes(32))); + + ObservabilityServer::tool(LeakyTool::class) + ->assertOk() + ->assertDontSee('bob@example.com') + ->assertSee('@example.com'); + }); +}); + +function mcpRedactor(?string $profile = null): McpResponseRedactor +{ + return new McpResponseRedactor(resolve(Redactor::class), $profile); +} + +describe('McpResponseRedactor on raw JSON-RPC responses', function (): void { + it('redacts each response of a streamed iterable as it is yielded', function (): void { + $responses = (function (): Generator { + yield JsonRpcResponse::result(1, ['content' => [['type' => 'text', 'text' => 'first bob@example.com']]]); + yield JsonRpcResponse::result(2, ['content' => [['type' => 'text', 'text' => 'second sk_live_4eC39HqLyjWDarjtT1zdp7dc']]]); + })(); + + $out = mcpRedactor()->redact($responses); + + expect($out)->toBeInstanceOf(Generator::class); + + $texts = array_map( + fn (JsonRpcResponse $r): string => $r->content['result']['content'][0]['text'], + iterator_to_array($out, false) + ); + + expect($texts[0])->toBe('first [REDACTED]') + ->and($texts[1])->not->toContain('sk_live_4eC39HqLyjWDarjtT1zdp7dc') + ->and($texts[1])->toStartWith('second '); + }); + + it('redacts a JSON-RPC error message', function (): void { + $response = mcpRedactor()->redact(JsonRpcResponse::error(1, -32000, 'failed for bob@example.com')); + + expect($response->content['error']['message'])->toBe('failed for [REDACTED]'); + }); + + it('redacts the content a streamed notification carries', function (): void { + $response = mcpRedactor()->redact(JsonRpcResponse::notification('notifications/message', [ + 'content' => [['type' => 'text', 'text' => 'hi bob@example.com']], + ])); + + expect($response->content['params']['content'][0]['text'])->toBe('hi [REDACTED]'); + }); + + it('redacts a prompt message given as a bare string and leaves content that is not a block alone', function (): void { + $response = mcpRedactor()->redact(JsonRpcResponse::result(1, [ + 'messages' => [ + ['role' => 'user', 'content' => 'ask bob@example.com'], + ['role' => 'assistant', 'content' => 42], + ], + 'content' => ['not a block', ['type' => 'text', 'text' => 'from bob@example.com']], + ])); + + $result = $response->content['result']; + + expect($result['messages'][0]['content'])->toBe('ask [REDACTED]') + ->and($result['messages'][1]['content'])->toBe(42) + ->and($result['content'][0])->toBe('not a block') + ->and($result['content'][1]['text'])->toBe('from [REDACTED]'); + }); + + it('redacts the text of a resource embedded in tool content', function (): void { + $response = mcpRedactor()->redact(JsonRpcResponse::result(1, [ + 'content' => [['type' => 'resource', 'resource' => ['uri' => 'file:///owner.txt', 'text' => 'owner bob@example.com']]], + ])); + + expect($response->content['result']['content'][0]['resource']['text'])->toBe('owner [REDACTED]'); + }); + + it('replaces structured content wholesale when it cannot be redacted', function (): void { + $response = mcpRedactor('no_such_profile')->redact(JsonRpcResponse::result(1, [ + 'structuredContent' => ['email' => 'bob@example.com'], + ])); + + expect($response->content['result']['structuredContent'])->toBe(['redaction' => 'failed']); + }); +}); diff --git a/tests/Feature/RedactResponseMiddlewareTest.php b/tests/Feature/RedactResponseMiddlewareTest.php new file mode 100644 index 0000000..fd87119 --- /dev/null +++ b/tests/Feature/RedactResponseMiddlewareTest.php @@ -0,0 +1,133 @@ + response()->json(['id' => 7, 'email' => 'bob@example.com', 'password' => 'hunter2'])) + ->middleware('redact'); + + $this->getJson('/me') + ->assertOk() + ->assertExactJson(['id' => 7, 'email' => '[REDACTED]', 'password' => '[REDACTED]']); + }); + + it('takes a profile', function (): void { + config()->set('app.key', 'base64:'.base64_encode(random_bytes(32))); + + Route::get('/me', fn () => response()->json(['contact' => 'alice@customer.com'])) + ->middleware('redact:observability'); + + $body = $this->getJson('/me')->assertOk()->json(); + + expect($body['contact'])->toMatch('/^u_[a-z0-9]+@customer\.com$/'); + }); + + it('redacts a plain text response as text', function (): void { + Route::get('/note', fn (): ResponseFactory|\Illuminate\Http\Response => response('contact bob@example.com', 200, ['Content-Type' => 'text/plain'])) + ->middleware('redact'); + + $this->get('/note')->assertOk()->assertSee('contact [REDACTED]', false); + }); + + it('redacts a JSON string body that is not a JsonResponse as data', function (): void { + Route::get('/raw', fn (): ResponseFactory|\Illuminate\Http\Response => response('{"password":"hunter2","n":1}', 200, ['Content-Type' => 'application/json'])) + ->middleware('redact'); + + $this->get('/raw')->assertOk()->assertExactJson(['password' => '[REDACTED]', 'n' => 1]); + }); + + it('leaves streamed and binary responses alone', function (): void { + $middleware = new RedactResponse(resolve(Redactor::class)); + $stream = new StreamedResponse(fn (): int => print ('bob@example.com')); + + expect($middleware->handle(Request::create('/'), fn (): StreamedResponse => $stream))->toBe($stream); + + $image = response('bob@example.com', 200, ['Content-Type' => 'image/png']); + + expect($middleware->handle(Request::create('/'), fn (): ResponseFactory|\Illuminate\Http\Response => $image)->getContent())->toBe('bob@example.com'); + }); + + it('fails closed when the profile does not exist', function (): void { + Route::get('/me', fn () => response()->json(['password' => 'hunter2'])) + ->middleware('redact:no_such_profile'); + + $response = $this->getJson('/me'); + + $response->assertStatus(500); + expect($response->getContent())->not->toContain('hunter2'); + }); + + it('keeps a typed field typed with the nullify operator', function (): void { + config()->set('redactor.profiles.api', array_merge(config('redactor.profiles.default'), [ + 'blocked_keys' => ['ssn', 'age'], + 'operators' => ['default' => 'redact', 'ssn' => 'nullify', 'age' => 'nullify'], + ])); + + Route::get('/me', fn () => response()->json(['name' => 'n', 'ssn' => '123-45-6789', 'age' => 42])) + ->middleware('redact:api'); + + $this->getJson('/me')->assertOk()->assertExactJson(['name' => 'n', 'ssn' => null, 'age' => null]); + }); +}); + +describe('The nullify operator', function (): void { + it('nulls a value found by its key, whatever its type, and reports it', function (): void { + config()->set('redactor.profiles.api', array_merge(config('redactor.profiles.default'), [ + 'blocked_keys' => ['secret'], + 'operators' => ['default' => 'nullify'], + 'mark_redacted' => false, + ])); + + $result = resolve(Redactor::class)->inspect([ + 'secret' => ['nested' => 'x'], + 'other' => ['secret' => 12], + ], 'api'); + + expect($result->value)->toBe(['secret' => null, 'other' => ['secret' => null]]) + ->and($result->redactedKeys)->toBe(['secret']); + }); + + it('nulls a value at a path', function (): void { + config()->set('redactor.profiles.api', array_merge(config('redactor.profiles.default'), [ + 'paths' => ['meta.score' => 'nullify'], + 'mark_redacted' => false, + ])); + + expect(resolve(Redactor::class)->redact(['meta' => ['score' => 9.5, 'ok' => true]], 'api')) + ->toBe(['meta' => ['score' => null, 'ok' => true]]); + }); + + it('deletes a span inside a string, since a string has no null to write', function (): void { + config()->set('redactor.profiles.api', array_merge(config('redactor.profiles.default'), [ + 'operators' => ['default' => 'redact', 'email' => 'nullify'], + 'mark_redacted' => false, + ])); + + expect(resolve(Redactor::class)->redact('mail bob@example.com now', 'api'))->toBe('mail now'); + }); +}); + +describe('The redact middleware and file downloads', function (): void { + it('leaves a file download alone rather than read the file into memory', function (): void { + $path = tempnam(sys_get_temp_dir(), 'redactor'); + file_put_contents($path, 'bob@example.com'); + $download = new BinaryFileResponse($path); + + $middleware = new RedactResponse(resolve(Redactor::class)); + + expect($middleware->handle(Request::create('/'), fn (): BinaryFileResponse => $download))->toBe($download); + + unlink($path); + }); +}); diff --git a/tests/Feature/RedactionPerformedEventTest.php b/tests/Feature/RedactionPerformedEventTest.php new file mode 100644 index 0000000..443e69d --- /dev/null +++ b/tests/Feature/RedactionPerformedEventTest.php @@ -0,0 +1,52 @@ +redact(['password' => 'hunter2', 'note' => 'mail bob@example.com and alice@example.com']); + + Event::assertDispatched(RedactionPerformed::class, function (RedactionPerformed $event): bool { + $serialised = json_encode($event); + + return $event->profile === 'default' + && $event->redactedKeys === ['password', 'note'] + && $event->rules === ['blocked_key' => 1, 'email' => 2] + && $event->entities === ['password' => 1, 'email' => 2] + && $event->findings === 3 + && ! str_contains((string) $serialised, 'hunter2') + && ! str_contains((string) $serialised, 'bob@example.com'); + }); + }); + + it('is not dispatched when nothing was redacted', function (): void { + Event::fake([RedactionPerformed::class]); + + resolve(Redactor::class)->redact(['plain' => 'text']); + + Event::assertNotDispatched(RedactionPerformed::class); + }); + + it('can be switched off', function (): void { + config()->set('redactor.events', false); + Event::fake([RedactionPerformed::class]); + + (new Redactor)->redact(['password' => 'x']); + + Event::assertNotDispatched(RedactionPerformed::class); + }); + + it('never lets a failing listener break redaction', function (): void { + Event::listen(RedactionPerformed::class, fn () => throw new \RuntimeException('metrics down')); + + expect(resolve(Redactor::class)->redact(['password' => 'x']))->toBe(['password' => '[REDACTED]', '_redacted' => true]); + }); +}); diff --git a/tests/Feature/RedactorAccuracyTest.php b/tests/Feature/RedactorAccuracyTest.php new file mode 100644 index 0000000..b2c852d --- /dev/null +++ b/tests/Feature/RedactorAccuracyTest.php @@ -0,0 +1,321 @@ + true, + 'strategies' => [ShannonEntropyStrategy::class], + 'safe_keys' => [], + 'blocked_keys' => [], + 'patterns' => [], + 'replacement' => '[REDACTED]', + 'mark_redacted' => false, + 'track_redacted_keys' => false, + 'non_redactable_object_behavior' => 'preserve', + 'max_value_length' => null, + 'redact_large_objects' => false, + 'max_object_size' => 100, + 'shannon_entropy' => [ + 'enabled' => true, + 'threshold' => 4.8, + 'min_length' => 20, + 'exclusion_patterns' => [], + ], + ], $overrides); +} + +describe('Multibyte-correct entropy', function (): void { + it('measures characters, not bytes', function (): void { + $strategy = new ShannonEntropyStrategy; + + // Four distinct characters, evenly distributed: exactly 2 bits. + // Measured over UTF-8 bytes each of these is three bytes, several of + // them shared, which reports a quite different number. + expect($strategy->calculateShannonEntropy('日本語能'))->toBeGreaterThan(1.9) + ->and($strategy->calculateShannonEntropy('日本語能'))->toBeLessThan(2.1); + }); + + it('agrees with the ASCII case for four distinct characters', function (): void { + $strategy = new ShannonEntropyStrategy; + + expect(round($strategy->calculateShannonEntropy('abcd'), 6)) + ->toBe(round($strategy->calculateShannonEntropy('日本語能'), 6)); + }); + + it('reports zero for a single repeated multibyte character', function (): void { + expect((new ShannonEntropyStrategy)->calculateShannonEntropy('日日日日日'))->toBe(0.0); + }); + + it('falls back to bytes for input that is not valid UTF-8', function (): void { + $binary = "\xff\xfe\x00\x01\xff\xfe"; + + expect((new ShannonEntropyStrategy)->calculateShannonEntropy($binary))->toBeGreaterThan(0.0); + }); + + it('counts min_length in characters', function (): void { + // 10 characters, 30 bytes. Judged by strlen it clears a 20-character + // minimum it should not reach. + config()->set('redactor.profiles.accuracy', accuracyProfile([ + 'shannon_entropy' => [ + 'enabled' => true, + 'threshold' => 0.5, + 'min_length' => 20, + 'exclusion_patterns' => [], + ], + ])); + + expect(resolve(Redactor::class)->redact(['t' => '日本語能力試験合格者'], 'accuracy')) + ->toBe(['t' => '日本語能力試験合格者']); + }); +}); + +describe('Per-charset entropy thresholds', function (): void { + it('catches a hex digest that a base64-shaped threshold misses', function (): void { + // A 40-char SHA-1 tops out at 4.0 bits per character, so a 4.8 + // threshold can never fire on one however random it is. + $digest = 'a94a8fe5ccb19ba61c4c0873d391e987982fbbd3'; + + config()->set('redactor.profiles.accuracy', accuracyProfile()); + expect(resolve(Redactor::class)->redact(['h' => $digest], 'accuracy'))->toBe(['h' => $digest]); + + config()->set('redactor.profiles.accuracy', accuracyProfile([ + 'shannon_entropy' => [ + 'enabled' => true, + 'threshold' => 4.8, + 'min_length' => 20, + 'charset_thresholds' => ['hex' => 3.0], + 'exclusion_patterns' => [], + ], + ])); + expect(resolve(Redactor::class)->redact(['h' => $digest], 'accuracy'))->toBe(['h' => '[REDACTED]']); + }); + + it('leaves the configured threshold in charge when no charset matches', function (): void { + config()->set('redactor.profiles.accuracy', accuracyProfile([ + 'shannon_entropy' => [ + 'enabled' => true, + 'threshold' => 4.8, + 'min_length' => 20, + 'charset_thresholds' => ['hex' => 3.0], + 'exclusion_patterns' => [], + ], + ])); + + // Contains '-', so neither hex nor base64: judged at 4.8. + expect(resolve(Redactor::class)->redact(['t' => 'sk-1234567890abcdef1234567890abcdef'], 'accuracy')) + ->toBe(['t' => 'sk-1234567890abcdef1234567890abcdef']); + }); + + it('never silently overrides an explicitly configured threshold', function (): void { + // No charset_thresholds configured means the single threshold applies + // to every token, whatever alphabet it uses. + config()->set('redactor.profiles.accuracy', accuracyProfile([ + 'shannon_entropy' => [ + 'enabled' => true, + 'threshold' => 3.0, + 'min_length' => 20, + 'exclusion_patterns' => [], + ], + ])); + + expect(resolve(Redactor::class)->redact(['h' => 'a94a8fe5ccb19ba61c4c0873d391e987982fbbd3'], 'accuracy')) + ->toBe(['h' => '[REDACTED]']); + }); + + it('identifies the alphabets it claims to', function (): void { + $strategy = new class extends ShannonEntropyStrategy + { + public function charsetOf(string $s): ?string + { + return $this->detectCharset($s); + } + }; + + expect($strategy->charsetOf('deadbeef0123'))->toBe('hex') + ->and($strategy->charsetOf('YWJjZGVmZ2hpams='))->toBe('base64') + ->and($strategy->charsetOf('abc-def_ghi'))->toBe('base64url') + ->and($strategy->charsetOf('has spaces here'))->toBeNull(); + }); +}); + +describe('Capture-group aware replacement', function (): void { + it('keeps the label and replaces only the secret', function (): void { + config()->set('redactor.profiles.capture', accuracyProfile([ + 'strategies' => [RegexPatternsStrategy::class], + 'shannon_entropy' => ['enabled' => false], + 'patterns' => [ + 'aws' => [ + 'pattern' => '/(aws_secret_access_key\s*=\s*)([A-Za-z0-9\/+]{40})/i', + 'capture' => 2, + ], + ], + ])); + + $secret = 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY'; + + expect(resolve(Redactor::class)->redact("aws_secret_access_key = {$secret}", 'capture')) + ->toBe('aws_secret_access_key = [REDACTED]'); + }); + + it('replaces the whole match when no capture group is declared', function (): void { + config()->set('redactor.profiles.capture', accuracyProfile([ + 'strategies' => [RegexPatternsStrategy::class], + 'shannon_entropy' => ['enabled' => false], + 'patterns' => ['aws' => '/aws_secret_access_key\s*=\s*[A-Za-z0-9\/+]{40}/i'], + ])); + + expect(resolve(Redactor::class)->redact('aws_secret_access_key = wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY', 'capture')) + ->toBe('[REDACTED]'); + }); + + it('combines capture groups with partial mode', function (): void { + config()->set('redactor.profiles.capture', accuracyProfile([ + 'strategies' => [RegexPatternsStrategy::class], + 'shannon_entropy' => ['enabled' => false], + 'patterns' => [ + 'card' => ['pattern' => '/(card:\s*)(\d{16})/', 'capture' => 2, 'mode' => 'partial', 'keep' => 4], + ], + ])); + + expect(resolve(Redactor::class)->redact('card: 4111111111111111 ok', 'capture')) + ->toBe('card: ************1111 ok'); + }); + + it('falls back to the whole match when the group did not participate', function (): void { + config()->set('redactor.profiles.capture', accuracyProfile([ + 'strategies' => [RegexPatternsStrategy::class], + 'shannon_entropy' => ['enabled' => false], + 'patterns' => [ + 'opt' => ['pattern' => '/secret(?:=(\w+))?/', 'capture' => 1], + ], + ])); + + expect(resolve(Redactor::class)->redact('bare secret here', 'capture')) + ->toBe('bare [REDACTED] here'); + }); +}); + +describe('Shipped file_scan patterns', function (): void { + it('no longer flags every 40-character alphanumeric run as an AWS key', function (): void { + // '/[0-9a-zA-Z\/+]{40}/' matched any SHA-1 digest, base64 chunk or + // minified identifier in the codebase. + $rule = RedactorConfig::fromConfig('file_scan')->patterns['aws_secret_key']; + + expect($rule->pattern)->not->toBe('/[0-9a-zA-Z\/+]{40}/') + ->and($rule->capture)->toBe(2); + + // The rule alone no longer fires on a bare digest. (Entropy detection + // may still flag it during a file scan - that is its job - but it is + // no longer reported as an AWS credential.) + expect(preg_match($rule->pattern, 'sha1 a94a8fe5ccb19ba61c4c0873d391e987982fbbd3caffe123')) + ->toBe(0) + ->and(preg_match($rule->pattern, 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY')) + ->toBe(0); + }); + + it('still catches an AWS secret key next to its label', function (): void { + $result = resolve(Redactor::class)->redact( + 'aws_secret_access_key = wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY', + 'file_scan' + ); + + expect($result)->toContain('[REDACTED]') + ->and($result)->not->toContain('wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY') + ->and($result)->toContain('aws_secret_access_key'); + }); + + it('keeps the password label while removing the value', function (): void { + $result = resolve(Redactor::class)->redact('DB_PASSWORD=sup3rs3cret', 'file_scan'); + + expect($result)->not->toContain('sup3rs3cret') + ->and(strtolower($result))->toContain('password'); + }); + + it('keeps the host while removing url credentials', function (): void { + $result = resolve(Redactor::class)->redact('https://admin:hunter2@db.example.com/x', 'file_scan'); + + expect($result)->not->toContain('hunter2') + ->and($result)->toContain('db.example.com'); + }); + + it('still catches unambiguous single-token secrets outright', function (): void { + foreach ([ + 'AKIAIOSFODNN7EXAMPLE', + 'ghp_1234567890abcdefghijklmnopqrstuvwxyz', + 'sk_test_1234567890abcdef1234567890abcdef', + ] as $secret) { + expect(resolve(Redactor::class)->redact("value: {$secret}", 'file_scan')) + ->not->toContain($secret); + } + }); +}); + +describe('The ASCII tokenise path matches the Unicode one', function (): void { + function tokenProfile(): array + { + return accuracyProfile([ + 'strategies' => [ShannonEntropyStrategy::class], + 'shannon_entropy' => [ + 'enabled' => true, + 'threshold' => 3.5, + 'min_length' => 12, + 'exclusion_patterns' => [], + ], + ]); + } + + it('finds the same token in an ASCII value', function (): void { + config()->set('redactor.profiles.tok', tokenProfile()); + + expect(resolve(Redactor::class)->redact(['t' => 'key Zx7Qm4Kd9Rb2Vn6Tp end'], 'tok')) + ->toBe(['t' => 'key [REDACTED] end']); + }); + + it('finds the same token when the value contains non-ASCII text', function (): void { + config()->set('redactor.profiles.tok', tokenProfile()); + + // Same secret, same neighbours, but the value is no longer ASCII - so + // the /u pattern is used and must reach the same answer. + expect(resolve(Redactor::class)->redact(['t' => 'клавиша Zx7Qm4Kd9Rb2Vn6Tp конец'], 'tok')) + ->toBe(['t' => 'клавиша [REDACTED] конец']); + }); + + it('splits on a Unicode space, which the ASCII pattern would not', function (): void { + config()->set('redactor.profiles.tok', tokenProfile()); + + // U+00A0 between the words: with /u these are two tokens and only the + // secret is replaced. Choosing the pattern per subject is what keeps + // this correct. + $value = "prefix\u{00A0}Zx7Qm4Kd9Rb2Vn6Tp"; + + $result = resolve(Redactor::class)->redact(['t' => $value], 'tok'); + + expect($result['t'])->toContain('prefix') + ->and($result['t'])->toContain('[REDACTED]') + ->and($result['t'])->not->toContain('Zx7Qm4Kd9Rb2Vn6Tp'); + }); + + it('recognises ASCII and non-ASCII subjects correctly', function (): void { + $strategy = new class extends ShannonEntropyStrategy + { + public function ascii(string $v): bool + { + return $this->isAscii($v); + } + }; + + expect($strategy->ascii('plain ascii text'))->toBeTrue() + ->and($strategy->ascii(''))->toBeTrue() + ->and($strategy->ascii('café'))->toBeFalse() + ->and($strategy->ascii("binary\xff\xfe"))->toBeFalse(); + }); +}); diff --git a/tests/Feature/RedactorAllowListTest.php b/tests/Feature/RedactorAllowListTest.php new file mode 100644 index 0000000..d91f54b --- /dev/null +++ b/tests/Feature/RedactorAllowListTest.php @@ -0,0 +1,199 @@ + true, + 'strategies' => [BlockedKeysStrategy::class, RegexPatternsStrategy::class, ShannonEntropyStrategy::class], + 'safe_keys' => [], + 'blocked_keys' => ['password'], + 'patterns' => [ + 'email' => ['pattern' => '/[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+/', 'entity' => 'email'], + ], + 'paths' => ['meta.contact' => 'redact'], + 'allowlist' => ['noreply@example.com', '/^test-\d+@example\.com$/'], + 'replacement' => '[REDACTED]', + 'mark_redacted' => false, + 'track_redacted_keys' => false, + 'non_redactable_object_behavior' => 'preserve', + 'max_value_length' => null, + 'redact_large_objects' => false, + 'max_object_size' => 100, + 'shannon_entropy' => ['enabled' => true, 'threshold' => 4.0, 'min_length' => 20, 'exclusion_patterns' => []], + ], $overrides); +} + +describe('Profile allowlist', function (): void { + beforeEach(fn () => config()->set('redactor.profiles.allow', allowProfile())); + + it('lets an allowed value through a pattern', function (): void { + expect(resolve(Redactor::class)->redact('from noreply@example.com and bob@example.com', 'allow')) + ->toBe('from noreply@example.com and [REDACTED]'); + }); + + it('compares literals case-insensitively and ignores surrounding whitespace', function (): void { + expect(resolve(Redactor::class)->redact('from NoReply@Example.COM', 'allow')) + ->toBe('from NoReply@Example.COM'); + }); + + it('accepts a regex entry', function (): void { + expect(resolve(Redactor::class)->redact('test-42@example.com and test-x@example.com', 'allow')) + ->toBe('test-42@example.com and [REDACTED]'); + }); + + it('lets an allowed value through a blocked key', function (): void { + config()->set('redactor.profiles.allow.allowlist', ['changeme']); + + $result = resolve(Redactor::class)->redact(['password' => 'changeme', 'other' => ['password' => 'hunter2']], 'allow'); + + expect($result['password'])->toBe('changeme') + ->and($result['other']['password'])->toBe('[REDACTED]'); + }); + + it('lets an allowed value through a path rule', function (): void { + config()->set('redactor.profiles.allow.allowlist', ['support']); + + $result = resolve(Redactor::class)->redact(['meta' => ['contact' => 'support']], 'allow'); + + expect($result['meta']['contact'])->toBe('support'); + }); + + it('lets an allowed value through the entropy detector', function (): void { + config()->set('redactor.profiles.allow.allowlist', ['Zx7Qm4Kd9Rb2Vn6Tp1Ws8Yc3Hf']); + + expect(resolve(Redactor::class)->redact('key Zx7Qm4Kd9Rb2Vn6Tp1Ws8Yc3Hf ok', 'allow')) + ->toBe('key Zx7Qm4Kd9Rb2Vn6Tp1Ws8Yc3Hf ok'); + }); + + it('does not report an allowed value as a finding', function (): void { + $result = resolve(Redactor::class)->inspect('noreply@example.com', 'allow'); + + expect($result->wasRedacted)->toBeFalse() + ->and($result->findings)->toBe([]); + }); + + it('never lets an unevaluatable regex entry allow anything', function (): void { + $list = AllowList::for(['/^\p{L}+$/u']); + + expect($list->allows("\xff\xfe"))->toBeFalse(); + }); + + it('treats a string that merely starts with a slash as a literal', function (): void { + $list = AllowList::for(['/var/log/app.log']); + + expect($list->allows('/var/log/app.log'))->toBeTrue() + ->and($list->allows('/var/log/other.log'))->toBeFalse(); + }); +}); + +describe('Per-rule allow', function (): void { + it('scopes the exception to the rule that declares it', function (): void { + config()->set('redactor.profiles.allow', allowProfile([ + 'allowlist' => [], + 'patterns' => [ + 'email' => [ + 'pattern' => '/[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+/', + 'allow' => ['/@example\.com$/'], + ], + 'token' => ['pattern' => '/tok_[a-z0-9]+/'], + ], + 'shannon_entropy' => ['enabled' => false], + ])); + + expect(resolve(Redactor::class)->redact('bob@example.com bob@customer.com tok_abc', 'allow')) + ->toBe('bob@example.com [REDACTED] [REDACTED]'); + }); +}); + +describe('Dictionary rules', function (): void { + it('redacts any listed word, longest first, case-insensitively', function (): void { + config()->set('redactor.profiles.allow', allowProfile([ + 'allowlist' => [], + 'patterns' => [ + 'codenames' => ['words' => ['Project Falcon', 'Falcon', 'Orion'], 'entity' => 'codename'], + ], + 'shannon_entropy' => ['enabled' => false], + ])); + + expect(resolve(Redactor::class)->redact('status of project falcon and ORION', 'allow')) + ->toBe('status of [REDACTED] and [REDACTED]'); + }); + + it('does not match inside a longer word', function (): void { + config()->set('redactor.profiles.allow', allowProfile([ + 'allowlist' => [], + 'patterns' => ['codenames' => ['words' => ['Orion']]], + 'shannon_entropy' => ['enabled' => false], + ])); + + expect(resolve(Redactor::class)->redact('Orionids are meteors', 'allow'))->toBe('Orionids are meteors'); + }); + + it('rejects an empty word list', function (): void { + config()->set('redactor.profiles.allow', allowProfile([ + 'patterns' => ['codenames' => ['words' => []]], + ])); + + resolve(Redactor::class)->redact('x', 'allow'); + })->throws(\InvalidArgumentException::class, 'words'); +}); + +describe('Entropy tokenising stays flat in memory', function (): void { + it('holds only tokens long enough to qualify', function (): void { + config()->set('redactor.profiles.allow', allowProfile([ + 'patterns' => [], + 'blocked_keys' => [], + 'shannon_entropy' => ['enabled' => true, 'threshold' => 4.8, 'min_length' => 25, 'exclusion_patterns' => []], + ])); + + $subject = str_repeat('lorem ipsum dolor sit amet consectetur ', 25_000); // ~1 MB of short words + $redactor = resolve(Redactor::class); + $redactor->redact('warm up', 'allow'); + + memory_reset_peak_usage(); + $before = memory_get_peak_usage(); + $redactor->redact($subject, 'allow'); + $delta = memory_get_peak_usage() - $before; + + // Every token is under min_length, so nothing should be collected at + // all; a few hundred KB of scratch is fine, ten times the input is not. + expect($delta)->toBeLessThan(strlen($subject)); + }); +}); + +describe('Allow list entries', function (): void { + it('ignores blank entries rather than allowing an empty value', function (): void { + $list = AllowList::for(['', ' ', 'noreply@example.com']); + + expect($list->allows(''))->toBeFalse() + ->and($list->allows('noreply@example.com'))->toBeTrue(); + }); + + it('treats an entry too short to be a regex as a literal', function (): void { + $list = AllowList::for(['//', '~']); + + expect($list->allows('//'))->toBeTrue() + ->and($list->allows('~'))->toBeTrue() + ->and($list->allows('anything'))->toBeFalse(); + }); + + it('accepts bracket-delimited regex entries', function (): void { + $list = AllowList::for(['(^test-\d+$)i', '[^sandbox-\w+$]', '{^demo-\d+$}', '<^sample-\d+$>']); + + expect($list->allows('TEST-42'))->toBeTrue() + ->and($list->allows('sandbox-abc'))->toBeTrue() + ->and($list->allows('demo-1'))->toBeTrue() + ->and($list->allows('sample-2'))->toBeTrue() + ->and($list->allows('test-x'))->toBeFalse(); + }); +}); diff --git a/tests/Feature/RedactorApiConventionsTest.php b/tests/Feature/RedactorApiConventionsTest.php new file mode 100644 index 0000000..69d4cc0 --- /dev/null +++ b/tests/Feature/RedactorApiConventionsTest.php @@ -0,0 +1,126 @@ +withoutMarkers()->redact(['password' => 'x', 'id' => 1]); + + expect($result)->toBe(['password' => '[REDACTED]', 'id' => 1]); + }); + + it('inspects', function (): void { + $result = Redactor::profile('default')->inspect(['password' => 'x']); + + expect($result)->toBeInstanceOf(RedactionResult::class) + ->and($result->redactedKeys)->toBe(['password']); + }); + + it('is conditionable and macroable', function (): void { + PendingRedaction::macro('strictly', fn () => $this->profile('strict')); + + $pending = Redactor::profile('default')->when(true, fn (PendingRedaction $p) => $p->strictly()); + + expect($pending)->toBeInstanceOf(PendingRedaction::class) + ->and($pending->redact(['name' => 'Bob'])['name'])->toBe('[REDACTED]'); + }); + + it('never throws from redactSafely', function (): void { + expect(Redactor::profile('nope')->redactSafely(['password' => 'x']))->toBe('[REDACTED] (redaction failed)'); + }); + + it('exposes inspect, profiles, hasProfile and strategies on the service', function (): void { + expect(Redactor::inspect('a@b.com')->wasRedacted)->toBeTrue() + ->and(Redactor::profiles())->toContain('default', 'strict') + ->and(Redactor::hasProfile('default'))->toBeTrue() + ->and(Redactor::hasProfile('nope'))->toBeFalse() + ->and(Redactor::strategies('default'))->each->toBeInstanceOf(Strategy::class); + }); + + it('is macroable and conditionable itself', function (): void { + RedactorService::macro('shout', fn (string $s): string => strtoupper($this->redact($s))); + + expect(Redactor::shout('hi a@b.com'))->toBe('HI [REDACTED]') + ->and(resolve(RedactorService::class)->when(false, fn () => throw new \LogicException))->toBeInstanceOf(RedactorService::class); + }); +}); + +describe('Results are array-friendly', function (): void { + it('serialises a result and its findings without the matched text', function (): void { + $result = Redactor::inspect(['email' => 'bob@example.com']); + + $array = $result->toArray(); + + expect($array['was_redacted'])->toBeTrue() + ->and($array['findings'][0]['rule'])->toBe('blocked_key') + ->and(json_encode($result))->not->toContain('bob@example.com') + ->and(json_decode((string) json_encode($result), true)['redacted_keys'])->toBe(['email']); + }); +}); + +describe('Package exceptions', function (): void { + it('throws a catchable package type for a missing profile', function (): void { + try { + Redactor::redact('x', 'nope'); + } catch (ProfileNotFoundException $e) { + expect($e)->toBeInstanceOf(RedactorException::class) + ->and($e)->toBeInstanceOf(ConfigurationException::class) + ->and($e)->toBeInstanceOf(\InvalidArgumentException::class) + ->and($e->getMessage())->toBe('Redaction profile [nope] is not configured.'); + + return; + } + + $this->fail('No exception thrown.'); + }); + + it('throws a configuration exception for a bad value, still an InvalidArgumentException', function (): void { + config()->set('redactor.profiles.default.max_depth', 'deep'); + + expect(fn () => Redactor::redact('x'))->toThrow(ConfigurationException::class, 'max_depth'); + }); + + it('throws a pseudonymization key exception for a short key', function (): void { + expect(fn (): Pseudonymizer => Pseudonymizer::fromKey('short'))->toThrow(PseudonymizationKeyException::class); + }); + + it('marks git failures', function (): void { + expect(new GitException('x'))->toBeInstanceOf(RedactorException::class); + }); +}); + +describe('Findings are JSON-friendly on their own', function (): void { + it('serialises a finding the same way as toArray, without the matched text', function (): void { + $finding = new MatchFinding(rule: 'email', key: 'contact', offset: 2, length: 3, matched: 'bob'); + + expect(json_decode((string) json_encode($finding), true))->toBe($finding->toArray()) + ->and((string) json_encode($finding))->not->toContain('bob'); + }); +}); + +describe('Markers through the fluent entry point', function (): void { + it('writes the markers when asked, whatever the profile says', function (): void { + config()->set('redactor.profiles.default.mark_redacted', false); + + $plain = Redactor::redact(['password' => 'hunter2']); + $marked = Redactor::profile('default')->withMarkers()->redact(['password' => 'hunter2']); + + expect($plain)->not->toHaveKey('_redacted') + ->and($marked['_redacted'])->toBeTrue(); + }); +}); diff --git a/tests/Feature/RedactorBlockedKeyOperatorTest.php b/tests/Feature/RedactorBlockedKeyOperatorTest.php new file mode 100644 index 0000000..cbc8f8e --- /dev/null +++ b/tests/Feature/RedactorBlockedKeyOperatorTest.php @@ -0,0 +1,80 @@ + true, + 'strategies' => [BlockedKeysStrategy::class, RegexPatternsStrategy::class], + 'safe_keys' => [], + 'blocked_keys' => [], + 'patterns' => [ + 'email' => ['pattern' => '/[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+/', 'entity' => 'email'], + ], + 'operators' => ['default' => 'redact'], + 'min_confidence' => 0.0, + 'replacement' => '[REDACTED]', + 'mark_redacted' => false, + 'track_redacted_keys' => false, + 'non_redactable_object_behavior' => 'preserve', + 'max_value_length' => null, + 'redact_large_objects' => false, + 'max_object_size' => 100, + 'pseudonymization' => ['key' => testPseudonymizationKey()], + 'shannon_entropy' => ['enabled' => false], + ], $overrides); +} + +describe('Blocked keys go through operators', function (): void { + it('applies the entity operator to a value found by its key', function (): void { + config()->set('redactor.profiles.blocked', blockedKeyProfile([ + 'blocked_keys' => ['email'], + 'operators' => ['default' => 'redact', 'email' => ['surrogate' => ['preserve_domain' => true]]], + ])); + + $result = resolve(Redactor::class)->redact(['email' => 'alice@customer.com'], 'blocked'); + + expect($result['email'])->toMatch('/^u_[a-z0-9]+@customer\.com$/'); + }); + + it('produces the same surrogate whether the key or the pattern found it', function (): void { + config()->set('redactor.profiles.blocked', blockedKeyProfile([ + 'blocked_keys' => ['email'], + 'operators' => ['default' => 'redact', 'email' => 'surrogate'], + ])); + + $result = resolve(Redactor::class)->redact([ + 'email' => 'alice@customer.com', + 'note' => 'from alice@customer.com', + ], 'blocked'); + + expect($result['note'])->toBe('from '.$result['email']); + }); + + it('still collapses a container under a blocked key to the replacement', function (): void { + config()->set('redactor.profiles.blocked', blockedKeyProfile([ + 'blocked_keys' => ['credentials'], + 'operators' => ['default' => 'hash'], + ])); + + $result = resolve(Redactor::class)->redact(['credentials' => ['user' => 'a', 'pass' => 'b']], 'blocked'); + + expect($result['credentials'])->toBe('[REDACTED]'); + }); + + it('reports the key finding with a certain score', function (): void { + config()->set('redactor.profiles.blocked', blockedKeyProfile(['blocked_keys' => ['password']])); + + $result = resolve(Redactor::class)->inspect(['password' => 'hunter2'], 'blocked'); + + expect($result->findings[0]->rule)->toBe('blocked_key') + ->and($result->findings[0]->confidence?->score)->toBe(1.0); + }); +}); diff --git a/tests/Feature/RedactorBoundaryTest.php b/tests/Feature/RedactorBoundaryTest.php new file mode 100644 index 0000000..41f15a8 --- /dev/null +++ b/tests/Feature/RedactorBoundaryTest.php @@ -0,0 +1,377 @@ +` could become `>=` without a single + * test noticing. + */ +function boundaryProfile(array $overrides = []): array +{ + return array_merge([ + 'enabled' => true, + 'strategies' => [LargeObjectStrategy::class, LargeStringStrategy::class], + 'safe_keys' => [], + 'blocked_keys' => [], + 'patterns' => [], + 'replacement' => '[REDACTED]', + 'mark_redacted' => false, + 'track_redacted_keys' => false, + 'non_redactable_object_behavior' => 'preserve', + 'max_value_length' => null, + 'redact_large_objects' => false, + 'max_object_size' => 100, + 'shannon_entropy' => ['enabled' => false], + ], $overrides); +} + +function arrayOf(int $size): array +{ + return array_fill_keys(array_map(fn (int $i): string => "k{$i}", range(1, $size)), 'v'); +} + +describe('max_object_size boundary', function (): void { + beforeEach(function (): void { + config()->set('redactor.profiles.boundary', boundaryProfile([ + 'redact_large_objects' => true, + 'max_object_size' => 10, + ])); + }); + + it('leaves an array of exactly max_object_size alone', function (): void { + $result = resolve(Redactor::class)->redact(['payload' => arrayOf(10)], 'boundary'); + + expect($result['payload'])->not->toHaveKey('_large_object_redacted') + ->and($result['payload'])->toHaveCount(10); + }); + + it('redacts an array one item over max_object_size', function (): void { + $result = resolve(Redactor::class)->redact(['payload' => arrayOf(11)], 'boundary'); + + expect($result['payload'])->toHaveKey('_large_object_redacted') + ->and($result['payload']['_large_object_redacted'])->toContain('11 items'); + }); + + it('reports the real item count in the marker', function (): void { + $result = resolve(Redactor::class)->redact(['payload' => arrayOf(25)], 'boundary'); + + expect($result['payload']['_large_object_redacted'])->toContain('25 items') + ->and($result['payload']['_large_object_redacted'])->not->toContain('24 items') + ->and($result['payload']['_large_object_redacted'])->not->toContain('26 items'); + }); + + it('does nothing at all when redact_large_objects is off', function (): void { + config()->set('redactor.profiles.boundary', boundaryProfile([ + 'redact_large_objects' => false, + 'max_object_size' => 10, + ])); + + expect(resolve(Redactor::class)->redact(['payload' => arrayOf(50)], 'boundary')['payload']) + ->toHaveCount(50); + }); +}); + +describe('max_value_length boundary', function (): void { + beforeEach(function (): void { + config()->set('redactor.profiles.boundary', boundaryProfile(['max_value_length' => 20])); + }); + + it('leaves a string of exactly max_value_length alone', function (): void { + $value = str_repeat('a', 20); + + expect(resolve(Redactor::class)->redact(['s' => $value], 'boundary'))->toBe(['s' => $value]); + }); + + it('redacts a string one character over max_value_length', function (): void { + $result = resolve(Redactor::class)->redact(['s' => str_repeat('a', 21)], 'boundary'); + + expect($result['s'])->toBe(str_repeat('a', 20).' [REDACTED] (String truncated: 21 characters, 20 kept)'); + }); + + it('reports the real length in the marker', function (): void { + $result = resolve(Redactor::class)->redact(['s' => str_repeat('a', 500)], 'boundary'); + + expect($result['s'])->toContain('500 characters') + ->and($result['s'])->not->toContain('499 characters') + ->and($result['s'])->not->toContain('501 characters'); + }); + + it('marks the payload as redacted when the limit trips', function (): void { + config()->set('redactor.profiles.boundary', boundaryProfile([ + 'max_value_length' => 20, + 'mark_redacted' => true, + 'track_redacted_keys' => true, + ])); + + $result = resolve(Redactor::class)->inspect(['s' => str_repeat('a', 21)], 'boundary'); + + expect($result->wasRedacted)->toBeTrue() + ->and($result->redactedKeys)->toBe(['s']); + }); +}); + +describe('entropy threshold and length boundaries', function (): void { + it('skips a token one character under min_length', function (): void { + config()->set('redactor.profiles.boundary', boundaryProfile([ + 'strategies' => [ShannonEntropyStrategy::class], + 'shannon_entropy' => [ + 'enabled' => true, + 'threshold' => 1.0, + 'min_length' => 16, + 'exclusion_patterns' => [], + ], + ])); + + $short = 'Zx7Qm4Kd9Rb2Vn6'; // 15 characters + + expect(strlen($short))->toBe(15) + ->and(resolve(Redactor::class)->redact(['t' => $short], 'boundary'))->toBe(['t' => $short]); + }); + + it('inspects a token of exactly min_length', function (): void { + config()->set('redactor.profiles.boundary', boundaryProfile([ + 'strategies' => [ShannonEntropyStrategy::class], + 'shannon_entropy' => [ + 'enabled' => true, + 'threshold' => 1.0, + 'min_length' => 16, + 'exclusion_patterns' => [], + ], + ])); + + $exact = 'Zx7Qm4Kd9Rb2Vn6T'; // 16 characters + + expect(strlen($exact))->toBe(16) + ->and(resolve(Redactor::class)->redact(['t' => $exact], 'boundary'))->toBe(['t' => '[REDACTED]']); + }); + + it('redacts at exactly the threshold, not only above it', function (): void { + $token = 'abcdefgh'; // 8 distinct characters => exactly 3.0 bits + $entropy = (new ShannonEntropyStrategy)->calculateShannonEntropy($token); + + expect($entropy)->toBe(3.0); + + config()->set('redactor.profiles.boundary', boundaryProfile([ + 'strategies' => [ShannonEntropyStrategy::class], + 'shannon_entropy' => [ + 'enabled' => true, + 'threshold' => 3.0, + 'min_length' => 8, + 'exclusion_patterns' => [], + ], + ])); + + expect(resolve(Redactor::class)->redact(['t' => $token], 'boundary'))->toBe(['t' => '[REDACTED]']); + }); + + it('leaves a token just under the threshold alone', function (): void { + $token = 'abcdefgh'; + + config()->set('redactor.profiles.boundary', boundaryProfile([ + 'strategies' => [ShannonEntropyStrategy::class], + 'shannon_entropy' => [ + 'enabled' => true, + 'threshold' => 3.01, + 'min_length' => 8, + 'exclusion_patterns' => [], + ], + ])); + + expect(resolve(Redactor::class)->redact(['t' => $token], 'boundary'))->toBe(['t' => $token]); + }); + + it('does nothing at all when entropy detection is disabled', function (): void { + config()->set('redactor.profiles.boundary', boundaryProfile([ + 'strategies' => [ShannonEntropyStrategy::class], + 'shannon_entropy' => [ + 'enabled' => false, + 'threshold' => 0.1, + 'min_length' => 1, + 'exclusion_patterns' => [], + ], + ])); + + expect(resolve(Redactor::class)->redact(['t' => 'Zx7Qm4Kd9Rb2Vn6Tp1Ws8'], 'boundary')) + ->toBe(['t' => 'Zx7Qm4Kd9Rb2Vn6Tp1Ws8']); + }); +}); + +describe('max_depth boundary', function (): void { + it('walks exactly max_depth levels and replaces the next', function (): void { + config()->set('redactor.profiles.boundary', boundaryProfile([ + 'strategies' => [RegexPatternsStrategy::class], + 'patterns' => ['secret' => '/SECRET/'], + 'max_depth' => 3, + ])); + + // Depth 1 = the root array, 2 = 'a', 3 = 'b', 4 = 'c' (over the limit). + $result = resolve(Redactor::class)->redact( + ['a' => ['b' => ['c' => ['leaf' => 'SECRET']]]], + 'boundary' + ); + + expect($result['a']['b']['c'])->toBe('[REDACTED] (Max depth of 3 exceeded)'); + }); + + it('reaches a leaf sitting exactly at max_depth', function (): void { + config()->set('redactor.profiles.boundary', boundaryProfile([ + 'strategies' => [RegexPatternsStrategy::class], + 'patterns' => ['secret' => '/SECRET/'], + 'max_depth' => 3, + ])); + + $result = resolve(Redactor::class)->redact(['a' => ['b' => ['leaf' => 'SECRET']]], 'boundary'); + + expect($result['a']['b']['leaf'])->toBe('[REDACTED]'); + }); +}); + +describe('partial mode keep boundary', function (): void { + beforeEach(function (): void { + config()->set('redactor.profiles.boundary', boundaryProfile([ + 'strategies' => [RegexPatternsStrategy::class], + 'patterns' => ['digits' => ['pattern' => '/\d+/', 'mode' => PatternRule::MODE_PARTIAL, 'keep' => 4]], + ])); + }); + + it('masks everything when the match is exactly keep characters', function (): void { + expect(resolve(Redactor::class)->redact('1234', 'boundary'))->toBe('****'); + }); + + it('reveals the tail as soon as the match is one character longer', function (): void { + expect(resolve(Redactor::class)->redact('12345', 'boundary'))->toBe('*2345'); + }); + + it('masks a match shorter than keep entirely', function (): void { + expect(resolve(Redactor::class)->redact('12', 'boundary'))->toBe('**'); + }); +}); + +describe('Luhn length window boundaries', function (): void { + it('rejects 11 digits and accepts a valid 12', function (): void { + // 12 is the shortest real card length (Maestro); anything shorter is a + // sequence number that happens to pass the checksum. + expect(Validator::luhn('00000000000'))->toBeFalse() + ->and(strlen('000000000000'))->toBe(12) + ->and(Validator::luhn('000000000000'))->toBeTrue(); + }); + + it('accepts 19 digits and rejects 20', function (): void { + expect(Validator::luhn('0000000000000000000'))->toBeTrue() + ->and(Validator::luhn('00000000000000000000'))->toBeFalse(); + }); +}); + +describe('The entropy length gate is a shortcut, not a behaviour change', function (): void { + function gatedProfile(int $minLength, float $threshold = 1.0): array + { + return boundaryProfile([ + 'strategies' => [ShannonEntropyStrategy::class], + 'shannon_entropy' => [ + 'enabled' => true, + 'threshold' => $threshold, + 'min_length' => $minLength, + 'exclusion_patterns' => [], + ], + ]); + } + + it('still inspects a value of exactly min_length', function (): void { + config()->set('redactor.profiles.boundary', gatedProfile(16)); + + $exact = 'Zx7Qm4Kd9Rb2Vn6T'; // 16 characters + + expect(strlen($exact))->toBe(16) + ->and(resolve(Redactor::class)->redact(['t' => $exact], 'boundary'))->toBe(['t' => '[REDACTED]']); + }); + + it('still finds a long token inside a long value', function (): void { + config()->set('redactor.profiles.boundary', gatedProfile(20)); + + $result = resolve(Redactor::class)->redact( + ['t' => 'deploy used Zx7Qm4Kd9Rb2Vn6Tp1Ws8Yc3Hf then finished'], + 'boundary' + ); + + expect($result['t'])->toBe('deploy used [REDACTED] then finished'); + }); + + it('skips a value that cannot contain a long enough token', function (): void { + config()->set('redactor.profiles.boundary', gatedProfile(30)); + + // Every token is short, and so is the value. + expect(resolve(Redactor::class)->redact(['t' => 'a b c d e f'], 'boundary')) + ->toBe(['t' => 'a b c d e f']); + }); + + it('counts characters, not bytes, once the byte gate passes', function (): void { + // 10 characters but 30 bytes: the byte count lets it through, and the + // character count must then reject it. + config()->set('redactor.profiles.boundary', gatedProfile(20, 0.5)); + + $multibyte = '日本語能力試験合格者'; // 10 chars, 30 bytes + + expect(strlen($multibyte))->toBe(30) + ->and(mb_strlen($multibyte))->toBe(10) + ->and(resolve(Redactor::class)->redact(['t' => $multibyte], 'boundary')) + ->toBe(['t' => $multibyte]); + }); + + it('inspects a multibyte value that is genuinely long enough', function (): void { + config()->set('redactor.profiles.boundary', gatedProfile(8, 2.0)); + + $multibyte = '日本語能力試験合格'; // 9 characters + + expect(mb_strlen($multibyte))->toBe(9) + ->and(resolve(Redactor::class)->redact(['t' => $multibyte], 'boundary')) + ->toBe(['t' => '[REDACTED]']); + }); + + it('rejects a non-numeric min_length at config time', function (): void { + config()->set('redactor.profiles.boundary', boundaryProfile([ + 'strategies' => [ShannonEntropyStrategy::class], + 'shannon_entropy' => [ + 'enabled' => true, + 'threshold' => 1.0, + 'min_length' => 'lots', + 'exclusion_patterns' => [], + ], + ])); + + expect(fn (): RedactorConfig => RedactorConfig::fromConfig('boundary')) + ->toThrow(\InvalidArgumentException::class, 'shannon_entropy.min_length'); + }); + + it('analyses everything when a hand-built config carries no usable min_length', function (): void { + // The gate is defensive as well as fast: a config assembled directly, + // bypassing validation, must fall through to analysis rather than + // silently skipping every value. + $strategy = new class extends ShannonEntropyStrategy + { + public function isTooShort(string $s, array $cfg): bool + { + return $this->tooShort($s, $cfg); + } + }; + + expect($strategy->isTooShort('short', ['min_length' => 'lots']))->toBeFalse() + ->and($strategy->isTooShort('short', []))->toBeTrue() + ->and($strategy->isTooShort(str_repeat('a', 40), []))->toBeFalse(); + }); +}); diff --git a/tests/Feature/RedactorConfidenceTest.php b/tests/Feature/RedactorConfidenceTest.php new file mode 100644 index 0000000..40dfaba --- /dev/null +++ b/tests/Feature/RedactorConfidenceTest.php @@ -0,0 +1,273 @@ + true, + 'strategies' => [RegexPatternsStrategy::class], + 'safe_keys' => [], + 'blocked_keys' => [], + 'patterns' => $patterns, + 'paths' => [], + 'operators' => ['default' => 'redact'], + 'min_confidence' => 0.0, + 'replacement' => '[REDACTED]', + 'mark_redacted' => false, + 'track_redacted_keys' => false, + 'non_redactable_object_behavior' => 'preserve', + 'max_value_length' => null, + 'redact_large_objects' => false, + 'max_object_size' => 100, + 'shannon_entropy' => ['enabled' => false], + ], $overrides); +} + +describe('Confidence arithmetic', function (): void { + it('never exceeds certainty however many signals stack', function (): void { + $confidence = Confidence::of(0.6); + + for ($i = 0; $i < 50; $i++) { + $confidence = $confidence->with("s{$i}", 0.5, 'another signal'); + } + + expect($confidence->score)->toBeLessThanOrEqual(1.0) + ->and($confidence->score)->toBeGreaterThan(0.99); + }); + + it('applies a positive signal to the remaining headroom, not flat', function (): void { + // Flat addition would let two 0.6 signals claim 1.2 certainty, and + // would let one strong signal swamp everything after it. + $once = Confidence::of(0.5)->with('a', 0.5, 'r'); + $twice = $once->with('b', 0.5, 'r'); + + expect($once->score)->toBe(0.75) + ->and($twice->score)->toBe(0.875); + }); + + it('reduces the score for a negative signal', function (): void { + expect(Confidence::of(0.8)->with('a', -0.5, 'r')->score) + ->toBeLessThan(0.8); + }); + + it('clamps a base outside the range', function (): void { + expect(Confidence::of(5.0)->score)->toBe(1.0) + ->and(Confidence::of(-5.0)->score)->toBe(0.0); + }); + + it('explains every contribution', function (): void { + $confidence = Confidence::of(0.6, 'pattern matched')->with('luhn', 0.75, 'checksum passed'); + + expect($confidence->explain())->toHaveCount(2) + ->and($confidence->explain()[1])->toContain('luhn') + ->and($confidence->explain()[1])->toContain('checksum passed'); + }); + + it('labels bands a human can sort by', function (): void { + expect(Confidence::of(0.95)->label())->toBe('high') + ->and(Confidence::of(0.7)->label())->toBe('medium') + ->and(Confidence::of(0.4)->label())->toBe('low') + ->and(Confidence::of(0.1)->label())->toBe('very-low'); + }); +}); + +describe('Scoring a detection', function (): void { + it('raises the score when a checksum passes', function (): void { + config()->set('redactor.profiles.conf', confidenceProfile([ + 'card' => ['pattern' => '/\b\d{16}\b/', 'confidence' => 0.3, 'validator' => 'luhn'], + ], ['track_redacted_keys' => true, 'mark_redacted' => true])); + + $result = resolve(Redactor::class)->inspect(['v' => '4111111111111111'], 'conf'); + + expect($result->findings[0]->confidence?->score)->toBeGreaterThan(0.3) + ->and(implode(' ', $result->findings[0]->confidence?->explain() ?? []))->toContain('luhn'); + }); + + it('raises the score when a credential keyword sits beside the match', function (): void { + config()->set('redactor.profiles.conf', confidenceProfile([ + 'token' => ['pattern' => '/[a-z0-9]{20,}/', 'confidence' => 0.3], + ])); + + $bare = resolve(Redactor::class)->inspect(['v' => 'abcdefghijklmnopqrstuvwxyz'], 'conf'); + $labelled = resolve(Redactor::class)->inspect(['v' => 'token=abcdefghijklmnopqrstuvwxyz'], 'conf'); + + expect($labelled->findings[0]->confidence?->score) + ->toBeGreaterThan($bare->findings[0]->confidence?->score ?? 1.0); + }); + + it('raises the score when the key itself names a credential', function (): void { + config()->set('redactor.profiles.conf', confidenceProfile([ + 'token' => ['pattern' => '/[a-z0-9]{20,}/', 'confidence' => 0.3], + ])); + + $neutral = resolve(Redactor::class)->inspect(['note' => 'abcdefghijklmnopqrstuvwxyz'], 'conf'); + $named = resolve(Redactor::class)->inspect(['api_key' => 'abcdefghijklmnopqrstuvwxyz'], 'conf'); + + expect($named->findings[0]->confidence?->score) + ->toBeGreaterThan($neutral->findings[0]->confidence?->score ?? 1.0); + }); + + it('ignores a keyword that only appears after the match', function (): void { + // " token" is usually the next field, not a label for this one. + config()->set('redactor.profiles.conf', confidenceProfile([ + 'token' => ['pattern' => '/^[a-z0-9]{20,}/', 'confidence' => 0.3], + ])); + + $after = resolve(Redactor::class)->inspect(['v' => 'abcdefghijklmnopqrstuvwxyz token'], 'conf'); + + expect($after->findings[0]->confidence?->score)->toBe(0.3); + }); +}); + +describe('The confidence floor', function (): void { + it('leaves a detection below the floor completely alone', function (): void { + config()->set('redactor.profiles.conf', confidenceProfile([ + 'weak' => ['pattern' => '/\bmaybe-\w+/', 'confidence' => 0.2], + ], ['min_confidence' => 0.5])); + + expect(resolve(Redactor::class)->redact(['v' => 'maybe-secret'], 'conf')) + ->toBe(['v' => 'maybe-secret']); + }); + + it('acts on the same detection once the floor drops', function (): void { + config()->set('redactor.profiles.conf', confidenceProfile([ + 'weak' => ['pattern' => '/\bmaybe-\w+/', 'confidence' => 0.2], + ], ['min_confidence' => 0.1])); + + expect(resolve(Redactor::class)->redact(['v' => 'maybe-secret'], 'conf')) + ->toBe(['v' => '[REDACTED]']); + }); + + it('does not report a filtered detection as a redaction', function (): void { + config()->set('redactor.profiles.conf', confidenceProfile([ + 'weak' => ['pattern' => '/\bmaybe-\w+/', 'confidence' => 0.2], + ], ['min_confidence' => 0.5])); + + expect(resolve(Redactor::class)->inspect(['v' => 'maybe-secret'], 'conf')->wasRedacted) + ->toBeFalse(); + }); + + it('lets a weak rule survive the floor when context corroborates it', function (): void { + // The whole point of scoring: the same pattern is noise on its own and + // a finding next to a keyword, without editing the pattern. + config()->set('redactor.profiles.conf', confidenceProfile([ + 'weak' => ['pattern' => '/[a-z0-9]{20,}/', 'confidence' => 0.3], + ], ['min_confidence' => 0.45])); + + expect(resolve(Redactor::class)->redact(['note' => 'abcdefghijklmnopqrstuvwxyz'], 'conf')) + ->toBe(['note' => 'abcdefghijklmnopqrstuvwxyz']); + + expect(resolve(Redactor::class)->redact(['note' => 'secret=abcdefghijklmnopqrstuvwxyz'], 'conf')) + ->toBe(['note' => 'secret=[REDACTED]']); + }); + + it('rejects a floor outside 0 to 1', function (): void { + config()->set('redactor.profiles.conf', confidenceProfile([], ['min_confidence' => 1.5])); + + expect(fn (): RedactorConfig => RedactorConfig::fromConfig('conf')) + ->toThrow(\InvalidArgumentException::class, 'min_confidence'); + }); +}); + +describe('Confidence in scan output', function (): void { + beforeEach(function (): void { + config(['redactor.scan.profile' => 'file_scan', 'redactor.scan.baseline' => null]); + + $this->dir = sys_get_temp_dir().'/redactor_conf_'.uniqid(); + mkdir($this->dir); + file_put_contents($this->dir.'/app.env', "AWS_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE\ncard 4111111111111111\n"); + }); + + afterEach(fn () => cleanupDirectory($this->dir)); + + it('reports a score, a severity and the signals behind it', function (): void { + Artisan::call('redactor:scan', ['paths' => [$this->dir], '--output' => 'json']); + + $finding = json_decode(Artisan::output(), true)[0]['findings'][0]; + + expect($finding)->toHaveKeys(['entity', 'confidence', 'severity', 'signals']) + ->and($finding['confidence'])->toBeFloat() + ->and($finding['severity'])->toBeIn(['high', 'medium', 'low', 'very-low']) + ->and($finding['signals'])->not->toBeEmpty(); + }); + + it('maps severity onto SARIF levels', function (): void { + Artisan::call('redactor:scan', ['paths' => [$this->dir], '--output' => 'sarif']); + + $results = json_decode(Artisan::output(), true)['runs'][0]['results']; + + foreach ($results as $result) { + expect($result['level'])->toBeIn(['error', 'warning', 'note']) + ->and($result['properties'])->toHaveKey('confidence'); + } + }); + + it('filters by --min-confidence', function (): void { + Artisan::call('redactor:scan', ['paths' => [$this->dir], '--output' => 'json']); + $all = count(json_decode(Artisan::output(), true)[0]['findings']); + + Artisan::call('redactor:scan', ['paths' => [$this->dir], '--output' => 'json', '--min-confidence' => '0.99']); + $strict = count(json_decode(Artisan::output(), true)[0]['findings'] ?? []); + + expect($all)->toBeGreaterThan(0) + ->and($strict)->toBeLessThanOrEqual($all); + }); + + it('rejects a --min-confidence outside 0 to 1', function (): void { + $exit = Artisan::call('redactor:scan', ['paths' => [$this->dir], '--min-confidence' => '7']); + + expect($exit)->toBe(1) + ->and(Artisan::output())->toContain('between 0 and 1'); + }); + + it('shows a severity column in the table', function (): void { + Artisan::call('redactor:scan', ['paths' => [$this->dir]]); + + expect(Artisan::output())->toContain('Severity'); + }); +}); + +describe('Low confidence in scan output', function (): void { + beforeEach(function (): void { + config(['redactor.scan.profile' => 'file_scan', 'redactor.scan.baseline' => null]); + config(['redactor.profiles.file_scan.min_confidence' => 0.0]); + config(['redactor.profiles.file_scan.patterns.weak' => ['pattern' => '/demo-secret-\d+/', 'confidence' => 0.4]]); + + $this->dir = sys_get_temp_dir().'/redactor_weak_'.uniqid(); + mkdir($this->dir); + file_put_contents($this->dir.'/weak.txt', "note demo-secret-12345\n"); + }); + + afterEach(fn () => cleanupDirectory($this->dir)); + + it('reports a weak rule as low severity', function (): void { + Artisan::call('redactor:scan', ['paths' => [$this->dir.'/weak.txt'], '--output' => 'json']); + + $finding = json_decode(Artisan::output(), true)[0]['findings'][0]; + + expect($finding['rule'])->toBe('weak') + ->and($finding['severity'])->toBe('low'); + }); + + it('maps low severity onto a SARIF note, so it never blocks a merge', function (): void { + Artisan::call('redactor:scan', ['paths' => [$this->dir.'/weak.txt'], '--output' => 'sarif']); + + expect(json_decode(Artisan::output(), true)['runs'][0]['results'][0]['level'])->toBe('note'); + }); + + it('labels low severity LOW in the table', function (): void { + Artisan::call('redactor:scan', ['paths' => [$this->dir.'/weak.txt']]); + + expect(Artisan::output())->toContain('LOW') + ->and(Artisan::output())->not->toContain('VERY LOW'); + }); +}); diff --git a/tests/Feature/RedactorConfigTest.php b/tests/Feature/RedactorConfigTest.php index 2055bdb..4778fd5 100644 --- a/tests/Feature/RedactorConfigTest.php +++ b/tests/Feature/RedactorConfigTest.php @@ -4,19 +4,23 @@ namespace Tests\Feature; +use Kirschbaum\Redactor\Exceptions\ConfigurationException; +use Kirschbaum\Redactor\Exceptions\ProfileNotFoundException; use Kirschbaum\Redactor\Redactor; use Kirschbaum\Redactor\RedactorConfig; +use Kirschbaum\Redactor\Strategies\BlockedKeysStrategy; +use Kirschbaum\Redactor\Strategies\SafeKeysStrategy; -describe('Redactor Configuration Tests', function () { - beforeEach(function () { +describe('Redactor Configuration Tests', function (): void { + beforeEach(function (): void { // Set up basic profile structure for tests config()->set('redactor.default_profile', 'default'); }); - it('can be disabled via configuration', function () { + it('can be disabled via configuration', function (): void { config()->set('redactor.profiles.default', [ 'enabled' => false, - 'strategies' => [\Kirschbaum\Redactor\Strategies\BlockedKeysStrategy::class], + 'strategies' => [BlockedKeysStrategy::class], 'safe_keys' => [], 'blocked_keys' => ['password'], 'patterns' => [], @@ -42,10 +46,10 @@ expect($result)->toBe($context); }); - it('does not add redacted flag when mark_redacted is false', function () { + it('does not add redacted flag when mark_redacted is false', function (): void { config()->set('redactor.profiles.default', [ 'enabled' => true, - 'strategies' => [\Kirschbaum\Redactor\Strategies\BlockedKeysStrategy::class], + 'strategies' => [BlockedKeysStrategy::class], 'safe_keys' => [], 'blocked_keys' => ['password'], 'patterns' => [], @@ -69,17 +73,17 @@ }); }); -describe('RedactorConfig DTO Tests', function () { - beforeEach(function () { +describe('RedactorConfig DTO Tests', function (): void { + beforeEach(function (): void { config()->set('redactor.default_profile', 'default'); }); - it('creates config from Laravel configuration with defaults', function () { + it('creates config from Laravel configuration with defaults', function (): void { config()->set('redactor.profiles.default', [ 'enabled' => true, 'strategies' => [ - \Kirschbaum\Redactor\Strategies\SafeKeysStrategy::class, - \Kirschbaum\Redactor\Strategies\BlockedKeysStrategy::class, + SafeKeysStrategy::class, + BlockedKeysStrategy::class, ], 'safe_keys' => [], 'blocked_keys' => [], @@ -117,12 +121,12 @@ ->and($config->profile)->toBe('default'); }); - it('creates config with custom values and handles invalid patterns', function () { + it('creates config with custom values and handles invalid patterns', function (): void { config()->set('redactor.profiles.default', [ 'enabled' => true, 'strategies' => [ - \Kirschbaum\Redactor\Strategies\SafeKeysStrategy::class, - \Kirschbaum\Redactor\Strategies\BlockedKeysStrategy::class, + SafeKeysStrategy::class, + BlockedKeysStrategy::class, ], 'safe_keys' => ['ID', 'User_ID'], 'blocked_keys' => ['PASSWORD', 'Secret'], @@ -145,17 +149,19 @@ expect($config->safeKeys)->toBe(['id', 'user_id']) // Converted to lowercase ->and($config->blockedKeys)->toBe(['password', 'secret']) // Converted to lowercase - ->and($config->patterns)->toBe(['valid' => '/valid-pattern/', 'another_valid' => '/another-valid-pattern/']) // Invalid pattern filtered out + ->and(array_keys($config->patterns))->toBe(['valid', 'another_valid']) // Invalid pattern filtered out + ->and($config->patterns['valid']->pattern)->toBe('/valid-pattern/') + ->and($config->patterns['another_valid']->pattern)->toBe('/another-valid-pattern/') ->and($config->replacement)->toBe('[CUSTOM]') ->and($config->maxValueLength)->toBe(100) ->and($config->trackRedactedKeys)->toBeTrue() ->and($config->nonRedactableObjectBehavior)->toBe('remove'); }); - it('handles non-integer max_value_length config', function () { + it('rejects a non-numeric max_value_length instead of silently disabling it', function (): void { config()->set('redactor.profiles.default', [ 'enabled' => true, - 'strategies' => [\Kirschbaum\Redactor\Strategies\SafeKeysStrategy::class], + 'strategies' => [SafeKeysStrategy::class], 'safe_keys' => [], 'blocked_keys' => [], 'patterns' => [], @@ -169,26 +175,27 @@ 'shannon_entropy' => ['enabled' => false], ]); - $config = RedactorConfig::fromConfig(); - - expect($config->maxValueLength)->toBeNull(); + // Previously this fell back to null, silently switching off the length + // cap the operator had asked for. + expect(fn (): RedactorConfig => RedactorConfig::fromConfig()) + ->toThrow(\InvalidArgumentException::class, 'profiles.default.max_value_length'); }); - it('throws exception for non-existent profile', function () { + it('throws exception for non-existent profile', function (): void { config()->set('redactor.profiles', []); // Empty profiles - expect(fn () => RedactorConfig::fromConfig('non_existent')) - ->toThrow(\InvalidArgumentException::class, "Redaction profile 'non_existent' not found in configuration."); + expect(fn (): RedactorConfig => RedactorConfig::fromConfig('non_existent')) + ->toThrow(ProfileNotFoundException::class, 'Redaction profile [non_existent] is not configured.'); }); - it('can list available profiles', function () { + it('can list available profiles', function (): void { config()->set('redactor.profiles', [ 'default' => ['enabled' => true], 'strict' => ['enabled' => true], 'performance' => ['enabled' => true], ]); - $profiles = RedactorConfig::getAvailableProfiles(); + $profiles = RedactorConfig::profiles(); expect($profiles)->toBeArray() ->and($profiles)->toContain('default') @@ -197,69 +204,72 @@ ->and($profiles)->toHaveCount(3); }); - it('checks if profile exists', function () { + it('checks if profile exists', function (): void { config()->set('redactor.profiles', [ 'default' => ['enabled' => true], 'strict' => ['enabled' => true], ]); - expect(RedactorConfig::profileExists('default'))->toBeTrue() - ->and(RedactorConfig::profileExists('strict'))->toBeTrue() - ->and(RedactorConfig::profileExists('non_existent'))->toBeFalse(); + expect(RedactorConfig::hasProfile('default'))->toBeTrue() + ->and(RedactorConfig::hasProfile('strict'))->toBeTrue() + ->and(RedactorConfig::hasProfile('non_existent'))->toBeFalse(); }); - it('throws exception for invalid profile configuration types', function () { + it('throws exception for invalid profile configuration types', function (): void { // Test when profile config is not an array config()->set('redactor.profiles.invalid_profile', 'not_an_array'); - expect(function () { + expect(function (): void { RedactorConfig::fromConfig('invalid_profile'); - })->toThrow(\InvalidArgumentException::class, "Invalid configuration for profile 'invalid_profile'"); + })->toThrow(ConfigurationException::class, 'Redaction profile [invalid_profile] must be an array.'); }); - it('handles zero and negative values in max value length validation', function () { - // Test validateMaxValueLength returning null for zero/negative values - config()->set('redactor.profiles.test_zero_max', [ - 'enabled' => true, - 'strategies' => [], - 'max_value_length' => 0, // Zero value should return null - 'safe_keys' => [], - 'blocked_keys' => [], - 'patterns' => [], - 'replacement' => '[REDACTED]', - 'mark_redacted' => true, - 'track_redacted_keys' => false, - 'non_redactable_object_behavior' => 'preserve', - 'redact_large_objects' => true, - 'max_object_size' => 100, - 'shannon_entropy' => ['enabled' => false], - ]); - - $config = RedactorConfig::fromConfig('test_zero_max'); - expect($config->maxValueLength)->toBeNull(); - - // Test with negative value - config()->set('redactor.profiles.test_negative_max', [ - 'enabled' => true, - 'strategies' => [], - 'max_value_length' => -5, // Negative value should return null - 'safe_keys' => [], - 'blocked_keys' => [], - 'patterns' => [], - 'replacement' => '[REDACTED]', - 'mark_redacted' => true, - 'track_redacted_keys' => false, - 'non_redactable_object_behavior' => 'preserve', - 'redact_large_objects' => true, - 'max_object_size' => 100, - 'shannon_entropy' => ['enabled' => false], - ]); + it('rejects zero and negative max_value_length', function (): void { + foreach ([0, -5, '0', '-5'] as $bad) { + config()->set('redactor.profiles.test_bad_max', [ + 'enabled' => true, + 'strategies' => [], + 'max_value_length' => $bad, + 'safe_keys' => [], + 'blocked_keys' => [], + 'patterns' => [], + 'replacement' => '[REDACTED]', + 'mark_redacted' => true, + 'track_redacted_keys' => false, + 'non_redactable_object_behavior' => 'preserve', + 'redact_large_objects' => true, + 'max_object_size' => 100, + 'shannon_entropy' => ['enabled' => false], + ]); + + expect(fn (): RedactorConfig => RedactorConfig::fromConfig('test_bad_max')) + ->toThrow(\InvalidArgumentException::class, 'profiles.test_bad_max.max_value_length'); + } + }); - $config2 = RedactorConfig::fromConfig('test_negative_max'); - expect($config2->maxValueLength)->toBeNull(); + it('treats null and an empty string as "no max_value_length"', function (): void { + foreach ([null, ''] as $disabled) { + config()->set('redactor.profiles.test_no_max', [ + 'enabled' => true, + 'strategies' => [], + 'max_value_length' => $disabled, + 'safe_keys' => [], + 'blocked_keys' => [], + 'patterns' => [], + 'replacement' => '[REDACTED]', + 'mark_redacted' => true, + 'track_redacted_keys' => false, + 'non_redactable_object_behavior' => 'preserve', + 'redact_large_objects' => true, + 'max_object_size' => 100, + 'shannon_entropy' => ['enabled' => false], + ]); + + expect(RedactorConfig::fromConfig('test_no_max')->maxValueLength)->toBeNull(); + } }); - it('validates regex patterns and removes invalid ones', function () { + it('validates regex patterns and removes invalid ones', function (): void { // Test validatePatterns with invalid regex patterns config()->set('redactor.profiles.test_invalid_patterns', [ 'enabled' => true, diff --git a/tests/Feature/RedactorContentTest.php b/tests/Feature/RedactorContentTest.php index fae1df4..4f49e21 100644 --- a/tests/Feature/RedactorContentTest.php +++ b/tests/Feature/RedactorContentTest.php @@ -5,7 +5,12 @@ use Kirschbaum\Redactor\RedactionContext; use Kirschbaum\Redactor\Redactor; use Kirschbaum\Redactor\RedactorConfig; -use Kirschbaum\Redactor\Strategies\RedactionStrategyInterface; +use Kirschbaum\Redactor\Strategies\BlockedKeysStrategy; +use Kirschbaum\Redactor\Strategies\Contracts\Strategy; +use Kirschbaum\Redactor\Strategies\LargeObjectStrategy; +use Kirschbaum\Redactor\Strategies\LargeStringStrategy; +use Kirschbaum\Redactor\Strategies\RegexPatternsStrategy; +use Kirschbaum\Redactor\Strategies\SafeKeysStrategy; use Kirschbaum\Redactor\Strategies\ShannonEntropyStrategy; // Simple test object without toArray method @@ -17,7 +22,7 @@ class SimpleTestObject public $prop3 = 'value3'; - public function getData() + public function getData(): array { return ['prop1' => $this->prop1, 'prop2' => $this->prop2, 'prop3' => $this->prop3]; } @@ -38,18 +43,18 @@ public function toArray(): array } } -describe('Redactor Content Tests', function () { - beforeEach(function () { +describe('Redactor Content Tests', function (): void { + beforeEach(function (): void { // Set up test configurations for different scenarios // This replaces the need for dynamic strategy addition/removal config()->set('redactor.profiles.no_shannon', [ 'enabled' => true, 'strategies' => [ - \Kirschbaum\Redactor\Strategies\SafeKeysStrategy::class, - \Kirschbaum\Redactor\Strategies\BlockedKeysStrategy::class, - \Kirschbaum\Redactor\Strategies\LargeObjectStrategy::class, - \Kirschbaum\Redactor\Strategies\LargeStringStrategy::class, - \Kirschbaum\Redactor\Strategies\RegexPatternsStrategy::class, + SafeKeysStrategy::class, + BlockedKeysStrategy::class, + LargeObjectStrategy::class, + LargeStringStrategy::class, + RegexPatternsStrategy::class, // Note: No ShannonEntropyStrategy ], 'safe_keys' => [], @@ -67,7 +72,7 @@ public function toArray(): array config()->set('redactor.profiles.test_shannon', [ 'enabled' => true, - 'strategies' => [\Kirschbaum\Redactor\Strategies\ShannonEntropyStrategy::class], + 'strategies' => [ShannonEntropyStrategy::class], 'safe_keys' => [], 'blocked_keys' => [], 'patterns' => [], @@ -89,12 +94,12 @@ public function toArray(): array config()->set('redactor.profiles.small_object_test', [ 'enabled' => true, 'strategies' => [ - \Kirschbaum\Redactor\Strategies\SafeKeysStrategy::class, - \Kirschbaum\Redactor\Strategies\BlockedKeysStrategy::class, - \Kirschbaum\Redactor\Strategies\LargeObjectStrategy::class, - \Kirschbaum\Redactor\Strategies\LargeStringStrategy::class, - \Kirschbaum\Redactor\Strategies\RegexPatternsStrategy::class, - \Kirschbaum\Redactor\Strategies\ShannonEntropyStrategy::class, + SafeKeysStrategy::class, + BlockedKeysStrategy::class, + LargeObjectStrategy::class, + LargeStringStrategy::class, + RegexPatternsStrategy::class, + ShannonEntropyStrategy::class, ], 'safe_keys' => [], 'blocked_keys' => [], @@ -110,11 +115,11 @@ public function toArray(): array ]); }); - test('it returns zero entropy when no ShannonEntropyStrategy is found during entropy calculation', function () { + test('it returns zero entropy when no ShannonEntropyStrategy is found during entropy calculation', function (): void { $redactor = new Redactor; // Use profile without ShannonEntropyStrategy - should return 0.0 - $strategies = $redactor->getStrategies('no_shannon'); + $strategies = $redactor->strategies('no_shannon'); $hasShannon = false; foreach ($strategies as $strategy) { if ($strategy instanceof ShannonEntropyStrategy) { @@ -125,36 +130,26 @@ public function toArray(): array expect($hasShannon)->toBeFalse(); - // calculateShannonEntropy uses default profile by default, so we need to check directly - // Since the method searches through strategies in the default profile, and our no_shannon profile - // doesn't have ShannonEntropyStrategy, we need to test with empty strategies - $entropy = $redactor->calculateShannonEntropy('high-entropy-string-xyz123'); + // Entropy is a property of the string, not of the profile: it stays the + // same whether or not the active profile lists ShannonEntropyStrategy. + $entropy = (new ShannonEntropyStrategy)->calculateShannonEntropy('high-entropy-string-xyz123'); + expect($entropy)->toBeGreaterThan(0.0); - // The default profile still has ShannonEntropyStrategy, so let's verify the logic - // by testing the actual case where no strategy is found - expect($entropy)->toBeGreaterThan(0.0); // Default profile has the strategy - - // Create redactor instance that specifically uses no_shannon profile for this test - // Since calculateShannonEntropy uses default profile, we test the edge case directly config()->set('redactor.default_profile', 'no_shannon'); - $noShannonRedactor = new Redactor; - $entropyNoStrategy = $noShannonRedactor->calculateShannonEntropy('high-entropy-string-xyz123'); - expect($entropyNoStrategy)->toBe(0.0); + expect((new ShannonEntropyStrategy)->calculateShannonEntropy('high-entropy-string-xyz123')) + ->toBe($entropy); }); - test('it returns false when no ShannonEntropyStrategy is found during pattern checking', function () { - // Set profile without ShannonEntropyStrategy as default + test('it reports no exclusion match when the profile configures no exclusion patterns', function (): void { config()->set('redactor.default_profile', 'no_shannon'); - $redactor = new Redactor; $config = RedactorConfig::fromConfig('no_shannon'); - // Should return false when no strategy found - $isCommon = $redactor->isCommonPattern('192.168.1.1', $config); + $isCommon = (new ShannonEntropyStrategy)->isCommonPattern('192.168.1.1', $config); expect($isCommon)->toBe(false); }); - test('it allows long hex strings to bypass common pattern exclusion for entropy checking', function () { + test('it allows long hex strings to bypass common pattern exclusion for entropy checking', function (): void { $redactor = new Redactor; $config = RedactorConfig::fromConfig('test_shannon'); @@ -162,24 +157,24 @@ public function toArray(): array $longHex = str_repeat('a1b2c3d4', 8); // 64 characters // This will exercise the specific branch where hex strings >= 32 continue - $isCommon = $redactor->isCommonPattern($longHex, $config); + $isCommon = (new ShannonEntropyStrategy)->isCommonPattern($longHex, $config); // The method should return false for long hex strings, allowing them to be entropy-checked expect($isCommon)->toBe(false); // Short hex should be considered common $shortHex = 'a1b2c3d4'; - $isCommonShort = $redactor->isCommonPattern($shortHex, $config); + $isCommonShort = (new ShannonEntropyStrategy)->isCommonPattern($shortHex, $config); expect($isCommonShort)->toBe(true); }); - test('it handles exceptions thrown by toArray method during object redaction', function () { + test('it handles exceptions thrown by toArray method during object redaction', function (): void { $redactor = new Redactor; // Create an object with a toArray method that throws an exception $objectWithBadToArray = new class { - public function toArray() + public function toArray(): never { throw new Exception('toArray failed'); } @@ -192,16 +187,16 @@ public function toArray() expect($result)->toHaveKey('bad_object'); }); - test('it skips large object redaction when feature is disabled', function () { + test('it skips large object redaction when feature is disabled', function (): void { // Create profile with large object redaction disabled config()->set('redactor.profiles.no_large_objects', [ 'enabled' => true, 'strategies' => [ - \Kirschbaum\Redactor\Strategies\SafeKeysStrategy::class, - \Kirschbaum\Redactor\Strategies\BlockedKeysStrategy::class, - \Kirschbaum\Redactor\Strategies\LargeStringStrategy::class, - \Kirschbaum\Redactor\Strategies\RegexPatternsStrategy::class, - \Kirschbaum\Redactor\Strategies\ShannonEntropyStrategy::class, + SafeKeysStrategy::class, + BlockedKeysStrategy::class, + LargeStringStrategy::class, + RegexPatternsStrategy::class, + ShannonEntropyStrategy::class, // Note: No LargeObjectStrategy ], 'safe_keys' => [], @@ -234,9 +229,9 @@ public function toArray() expect($result['large_data'])->toBe($largeArray); }); - test('it wraps non-array strategy results in redacted array structure', function () { + test('it wraps non-array strategy results in redacted array structure', function (): void { // Create a custom strategy that returns a string when processing arrays - $customStrategy = new class implements RedactionStrategyInterface + $customStrategy = new class implements Strategy { public function shouldHandle(mixed $value, string $key, RedactionContext $context): bool { @@ -260,8 +255,8 @@ public function handle(mixed $value, string $key, RedactionContext $context): mi 'enabled' => true, 'strategies' => [ 'array_strategy', // Custom strategy first - \Kirschbaum\Redactor\Strategies\SafeKeysStrategy::class, - \Kirschbaum\Redactor\Strategies\BlockedKeysStrategy::class, + SafeKeysStrategy::class, + BlockedKeysStrategy::class, ], 'safe_keys' => [], 'blocked_keys' => [], @@ -288,9 +283,9 @@ public function handle(mixed $value, string $key, RedactionContext $context): mi ->and($result['_redacted'])->toBeTrue(); }); - test('it removes keys when strategy returns removal signal', function () { + test('it removes keys when strategy returns removal signal', function (): void { // Create a custom strategy that removes specific keys by returning __REDACTOR_REMOVE_OBJECT__ - $removeStrategy = new class implements RedactionStrategyInterface + $removeStrategy = new class implements Strategy { public function shouldHandle(mixed $value, string $key, RedactionContext $context): bool { @@ -314,8 +309,8 @@ public function handle(mixed $value, string $key, RedactionContext $context): mi 'enabled' => true, 'strategies' => [ 'remove_strategy', // Custom strategy first - \Kirschbaum\Redactor\Strategies\SafeKeysStrategy::class, - \Kirschbaum\Redactor\Strategies\BlockedKeysStrategy::class, + SafeKeysStrategy::class, + BlockedKeysStrategy::class, ], 'safe_keys' => [], 'blocked_keys' => [], @@ -351,17 +346,17 @@ public function handle(mixed $value, string $key, RedactionContext $context): mi ->and($result['_redacted'])->toBeTrue(); }); - test('it handles large objects that exceed size limits during property counting', function () { + test('it handles large objects that exceed size limits during property counting', function (): void { // Create profile with very small max object size config()->set('redactor.profiles.small_object_test', [ 'enabled' => true, 'strategies' => [ - \Kirschbaum\Redactor\Strategies\SafeKeysStrategy::class, - \Kirschbaum\Redactor\Strategies\BlockedKeysStrategy::class, - \Kirschbaum\Redactor\Strategies\LargeObjectStrategy::class, - \Kirschbaum\Redactor\Strategies\LargeStringStrategy::class, - \Kirschbaum\Redactor\Strategies\RegexPatternsStrategy::class, - \Kirschbaum\Redactor\Strategies\ShannonEntropyStrategy::class, + SafeKeysStrategy::class, + BlockedKeysStrategy::class, + LargeObjectStrategy::class, + LargeStringStrategy::class, + RegexPatternsStrategy::class, + ShannonEntropyStrategy::class, ], 'safe_keys' => [], 'blocked_keys' => [], @@ -392,17 +387,17 @@ public function handle(mixed $value, string $key, RedactionContext $context): mi expect($message)->toContain('stdClass'); }); - test('it handles large objects detected via JSON encoding when toArray is unavailable', function () { + test('it handles large objects detected via JSON encoding when toArray is unavailable', function (): void { // Create profile with small max object size config()->set('redactor.profiles.json_size_test', [ 'enabled' => true, 'strategies' => [ - \Kirschbaum\Redactor\Strategies\SafeKeysStrategy::class, - \Kirschbaum\Redactor\Strategies\BlockedKeysStrategy::class, - \Kirschbaum\Redactor\Strategies\LargeObjectStrategy::class, - \Kirschbaum\Redactor\Strategies\LargeStringStrategy::class, - \Kirschbaum\Redactor\Strategies\RegexPatternsStrategy::class, - \Kirschbaum\Redactor\Strategies\ShannonEntropyStrategy::class, + SafeKeysStrategy::class, + BlockedKeysStrategy::class, + LargeObjectStrategy::class, + LargeStringStrategy::class, + RegexPatternsStrategy::class, + ShannonEntropyStrategy::class, ], 'safe_keys' => [], 'blocked_keys' => [], @@ -437,7 +432,7 @@ public function handle(mixed $value, string $key, RedactionContext $context): mi expect($result['large_obj'])->toHaveKey('_large_object_redacted'); }); - test('it handles objects with toArray method that throws exception during size detection', function () { + test('it handles objects with toArray method that throws exception during size detection', function (): void { config(['redactor.max_object_size' => 2]); $redactor = new Redactor; @@ -445,7 +440,7 @@ public function handle(mixed $value, string $key, RedactionContext $context): mi // Create object with toArray that throws an exception during size detection $largeObject = new class { - public function toArray() + public function toArray(): never { throw new Exception('Failed during detection'); } @@ -458,7 +453,7 @@ public function toArray() expect($result)->toHaveKey('large_obj'); }); - test('it handles objects with JSON encoding failures during size detection', function () { + test('it handles objects with JSON encoding failures during size detection', function (): void { config(['redactor.max_object_size' => 2]); $redactor = new Redactor; @@ -466,6 +461,9 @@ public function toArray() // Create object that will fail JSON encoding during size detection $largeObject = new class { + /** + * @var $this + */ public $circular; public function __construct() @@ -481,9 +479,9 @@ public function __construct() expect($result)->toHaveKey('large_obj'); }); - test('it returns strategy-processed objects directly when handled by custom strategies', function () { + test('it returns strategy-processed objects directly when handled by custom strategies', function (): void { // Create a custom strategy that specifically handles certain objects - $objectStrategy = new class implements RedactionStrategyInterface + $objectStrategy = new class implements Strategy { public function shouldHandle(mixed $value, string $key, RedactionContext $context): bool { @@ -507,8 +505,8 @@ public function handle(mixed $value, string $key, RedactionContext $context): mi 'enabled' => true, 'strategies' => [ 'object_strategy', // Custom strategy first - \Kirschbaum\Redactor\Strategies\SafeKeysStrategy::class, - \Kirschbaum\Redactor\Strategies\BlockedKeysStrategy::class, + SafeKeysStrategy::class, + BlockedKeysStrategy::class, ], 'safe_keys' => [], 'blocked_keys' => [], @@ -543,22 +541,23 @@ public function handle(mixed $value, string $key, RedactionContext $context): mi ->and($arrayResult['_redacted'])->toBeTrue(); }); - test('it returns all registered strategies via getStrategies method', function () { + test('it returns all registered strategies via getStrategies method', function (): void { $redactor = new Redactor; // Get the initial strategies from default profile - $strategies = $redactor->getStrategies(); + $strategies = $redactor->strategies(); - // Should have the 6 default strategies - expect($strategies)->toHaveCount(6); + // Seven of the eight configured strategies: entity recognition is + // configured but switched off, so it stays out of the chain. + expect($strategies)->toHaveCount(7); // Verify they are strategy instances foreach ($strategies as $strategy) { - expect($strategy)->toBeInstanceOf(RedactionStrategyInterface::class); + expect($strategy)->toBeInstanceOf(Strategy::class); } // Test with a custom profile that includes a registered custom strategy - $customStrategy = new class implements RedactionStrategyInterface + $customStrategy = new class implements Strategy { public function shouldHandle(mixed $value, string $key, RedactionContext $context): bool { @@ -577,12 +576,12 @@ public function handle(mixed $value, string $key, RedactionContext $context): mi config()->set('redactor.profiles.custom_strategy_test', [ 'enabled' => true, 'strategies' => [ - \Kirschbaum\Redactor\Strategies\SafeKeysStrategy::class, - \Kirschbaum\Redactor\Strategies\BlockedKeysStrategy::class, - \Kirschbaum\Redactor\Strategies\LargeObjectStrategy::class, - \Kirschbaum\Redactor\Strategies\LargeStringStrategy::class, - \Kirschbaum\Redactor\Strategies\RegexPatternsStrategy::class, - \Kirschbaum\Redactor\Strategies\ShannonEntropyStrategy::class, + SafeKeysStrategy::class, + BlockedKeysStrategy::class, + LargeObjectStrategy::class, + LargeStringStrategy::class, + RegexPatternsStrategy::class, + ShannonEntropyStrategy::class, 'test_strategy', // Custom strategy ], 'safe_keys' => [], @@ -599,15 +598,15 @@ public function handle(mixed $value, string $key, RedactionContext $context): mi ]); // Should now have 7 strategies when using custom profile - $strategiesWithCustom = $redactor->getStrategies('custom_strategy_test'); + $strategiesWithCustom = $redactor->strategies('custom_strategy_test'); expect($strategiesWithCustom)->toHaveCount(7); - // Profile without custom strategy should still have 6 - $strategiesDefault = $redactor->getStrategies(); - expect($strategiesDefault)->toHaveCount(6); + // Profile without custom strategy should still have 7 + $strategiesDefault = $redactor->strategies(); + expect($strategiesDefault)->toHaveCount(7); }); - test('it skips large object redaction when feature is disabled in configuration', function () { + test('it skips large object redaction when feature is disabled in configuration', function (): void { // Disable large object redaction config(['redactor.redact_large_objects' => false]); @@ -638,7 +637,7 @@ public function handle(mixed $value, string $key, RedactionContext $context): mi ->and($result)->not->toHaveKey('_redacted'); // No redaction occurred }); - test('it redacts large objects when they exceed size limits', function () { + test('it redacts large objects when they exceed size limits', function (): void { // Use the small_object_test profile we already set up $redactor = new Redactor; @@ -656,14 +655,20 @@ public function handle(mixed $value, string $key, RedactionContext $context): mi ->and($result['_redacted'])->toBeTrue(); }); - test('it handles objects with circular references gracefully', function () { + test('it handles objects with circular references gracefully', function (): void { $redactor = new Redactor; // Create an object with circular reference $problematicObject = new class { + /** + * @var $this + */ public $circular; + /** + * @var 'value1' + */ public $prop1; public function __construct() @@ -680,7 +685,7 @@ public function __construct() expect($result)->toHaveKey('problematic_obj'); }); - test('it handles string values correctly in redaction process', function () { + test('it handles string values correctly in redaction process', function (): void { $redactor = new Redactor; $data = ['simple_string' => 'test_value']; @@ -690,7 +695,7 @@ public function __construct() expect($result['simple_string'])->toBe('test_value'); }); - test('it handles objects with toArray method correctly', function () { + test('it handles objects with toArray method correctly', function (): void { $redactor = new Redactor; // Use an object that has a toArray method @@ -703,7 +708,7 @@ public function __construct() expect($result)->toHaveKey('test_object'); }); - test('it handles complex objects with encoding issues gracefully', function () { + test('it handles complex objects with encoding issues gracefully', function (): void { $redactor = new Redactor; // Create an object that might cause encoding issues @@ -713,6 +718,9 @@ public function __construct() public $prop2 = 'value2'; + /** + * @var $this + */ public $circular; public function __construct() @@ -728,11 +736,11 @@ public function __construct() expect($result)->toHaveKey('complex_obj'); }); - test('it allows long hex strings to bypass exclusion patterns in shannon entropy strategy', function () { + test('it allows long hex strings to bypass exclusion patterns in shannon entropy strategy', function (): void { // Create profile with Shannon entropy and hex exclusion pattern config()->set('redactor.profiles.hex_test', [ 'enabled' => true, - 'strategies' => [\Kirschbaum\Redactor\Strategies\ShannonEntropyStrategy::class], + 'strategies' => [ShannonEntropyStrategy::class], 'safe_keys' => [], 'blocked_keys' => [], 'patterns' => [], @@ -770,12 +778,12 @@ public function __construct() ->and($shortResult)->not->toHaveKey('_redacted'); }); - test('it calculates shannon entropy correctly', function () { + test('it calculates shannon entropy correctly', function (): void { $redactor = new Redactor; // Test the public calculateShannonEntropy method $highEntropyString = 'aB3$xY9#mK2@pL5!qR8%'; - $entropy = $redactor->calculateShannonEntropy($highEntropyString); + $entropy = (new ShannonEntropyStrategy)->calculateShannonEntropy($highEntropyString); // Should return a reasonable entropy value expect($entropy)->toBeGreaterThan(0.0) diff --git a/tests/Feature/RedactorCopyAvoidanceTest.php b/tests/Feature/RedactorCopyAvoidanceTest.php new file mode 100644 index 0000000..113214e --- /dev/null +++ b/tests/Feature/RedactorCopyAvoidanceTest.php @@ -0,0 +1,130 @@ + true, + 'strategies' => [BlockedKeysStrategy::class, RegexPatternsStrategy::class], + 'safe_keys' => [], + 'blocked_keys' => ['password'], + 'patterns' => ['email' => '/[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+/'], + 'paths' => [], + 'operators' => ['default' => 'redact'], + 'replacement' => '[REDACTED]', + 'mark_redacted' => false, + 'track_redacted_keys' => false, + 'non_redactable_object_behavior' => 'preserve', + 'max_value_length' => null, + 'redact_large_objects' => false, + 'max_object_size' => 1000, + 'shannon_entropy' => ['enabled' => false], + ], $overrides); +} + +describe('Returning the input when nothing changed', function (): void { + beforeEach(fn () => config()->set('redactor.profiles.copy', copyProfile())); + + it('returns a clean payload exactly as it arrived', function (): void { + $payload = [ + 'level' => 'info', + 'nested' => ['a' => 1, 'b' => ['c' => 'text']], + 'list' => [1, 2, 3], + ]; + + expect(resolve(Redactor::class)->redact($payload, 'copy'))->toBe($payload); + }); + + it('preserves key order and key types', function (): void { + $payload = ['z' => 1, 'a' => 2, 3 => 'three', 'm' => 4]; + + $result = resolve(Redactor::class)->redact($payload, 'copy'); + + expect(array_keys($result))->toBe(array_keys($payload)) + ->and($result)->toBe($payload); + }); + + it('keeps a list a list', function (): void { + $payload = ['a', 'b', 'c']; + + $result = resolve(Redactor::class)->redact($payload, 'copy'); + + expect(array_is_list($result))->toBeTrue() + ->and($result)->toBe($payload); + }); + + it('still redacts, and only what it should', function (): void { + $result = resolve(Redactor::class)->redact([ + 'keep' => 'ordinary', + 'password' => 'hunter2', + 'nested' => ['keep' => 'also ordinary', 'mail' => 'a@b.com'], + ], 'copy'); + + expect($result)->toBe([ + 'keep' => 'ordinary', + 'password' => '[REDACTED]', + 'nested' => ['keep' => 'also ordinary', 'mail' => '[REDACTED]'], + ]); + }); + + it('leaves untouched siblings alone when one branch changes', function (): void { + $payload = [ + 'untouched' => ['deep' => ['value' => 'nothing here']], + 'touched' => ['password' => 'hunter2'], + ]; + + $result = resolve(Redactor::class)->redact($payload, 'copy'); + + expect($result['untouched'])->toBe($payload['untouched']) + ->and($result['touched'])->toBe(['password' => '[REDACTED]']); + }); + + it('preserves order when a key is removed', function (): void { + config()->set('redactor.profiles.copy', copyProfile([ + 'paths' => ['b' => 'remove'], + ])); + + $result = resolve(Redactor::class)->redact(['a' => 1, 'b' => 2, 'c' => 3], 'copy'); + + expect($result)->toBe(['a' => 1, 'c' => 3]) + ->and(array_keys($result))->toBe(['a', 'c']); + }); + + it('removes a list entry without renumbering the rest', function (): void { + config()->set('redactor.profiles.copy', copyProfile([ + 'paths' => ['1' => 'remove'], + ])); + + $result = resolve(Redactor::class)->redact(['zero', 'one', 'two'], 'copy'); + + expect($result)->toBe([0 => 'zero', 2 => 'two']); + }); + + it('does not report a redaction for an untouched payload', function (): void { + $result = resolve(Redactor::class)->inspect(['a' => 'clean', 'b' => ['c' => 'also clean']], 'copy'); + + expect($result->wasRedacted)->toBeFalse() + ->and($result->findings)->toBe([]); + }); + + it('handles an empty array and an empty nested array', function (): void { + expect(resolve(Redactor::class)->redact([], 'copy'))->toBe([]) + ->and(resolve(Redactor::class)->redact(['a' => []], 'copy'))->toBe(['a' => []]); + }); + + it('does not mutate the array it was given', function (): void { + $payload = ['password' => 'hunter2', 'keep' => 'ordinary']; + $before = $payload; + + resolve(Redactor::class)->redact($payload, 'copy'); + + expect($payload)->toBe($before); + }); +}); diff --git a/tests/Feature/RedactorDetectionSeamTest.php b/tests/Feature/RedactorDetectionSeamTest.php new file mode 100644 index 0000000..0d49eb3 --- /dev/null +++ b/tests/Feature/RedactorDetectionSeamTest.php @@ -0,0 +1,405 @@ + true, + 'strategies' => [BlockedKeysStrategy::class, RegexPatternsStrategy::class, ShannonEntropyStrategy::class], + 'safe_keys' => [], + 'blocked_keys' => [], + 'patterns' => [ + 'stripe' => ['pattern' => '/sk_live_[A-Za-z0-9]{24}/', 'entity' => 'stripe_key', 'confidence' => 0.95], + 'email' => ['pattern' => SEAM_EMAIL, 'entity' => 'email'], + ], + 'operators' => ['default' => 'redact'], + 'min_confidence' => 0.0, + 'replacement' => '[REDACTED]', + 'mark_redacted' => false, + 'track_redacted_keys' => false, + 'non_redactable_object_behavior' => 'preserve', + 'max_value_length' => null, + 'redact_large_objects' => false, + 'max_object_size' => 100, + 'pseudonymization' => ['key' => testPseudonymizationKey()], + 'shannon_entropy' => [ + 'enabled' => true, + 'threshold' => 4.0, + 'min_length' => 20, + 'exclusion_patterns' => [], + ], + ], $overrides); +} + +function seamDetection(string $rule, int $offset, string $value, float $score = 0.6): Detection +{ + return new Detection(entity: $rule, rule: $rule, offset: $offset, value: $value, confidence: Confidence::of($score)); +} + +describe('Surrogates survive the rest of the chain', function (): void { + it('does not let the entropy detector eat a surrogate the regex detector just wrote', function (): void { + config()->set('redactor.profiles.seam', seamProfile([ + 'operators' => ['default' => 'redact', 'stripe_key' => ['surrogate' => ['preserve_prefix' => 8]]], + ])); + + $result = resolve(Redactor::class)->redact('key sk_live_4eC39HqLyjWDarjtT1zdp7dc end', 'seam'); + + expect($result)->toMatch('/^key sk_live_[A-Za-z0-9]{24} end$/') + ->and($result)->not->toContain('4eC39HqLyjWDarjtT1zdp7dc') + ->and($result)->not->toContain('[REDACTED]'); + }); + + it('reports the original secret, never the surrogate, in the findings', function (): void { + config()->set('redactor.profiles.seam', seamProfile([ + 'operators' => ['default' => 'redact', 'stripe_key' => 'surrogate'], + ])); + + $result = resolve(Redactor::class)->inspect('key sk_live_4eC39HqLyjWDarjtT1zdp7dc end', 'seam'); + + expect($result->findings)->toHaveCount(1) + ->and($result->findings[0]->rule)->toBe('stripe') + ->and($result->findings[0]->matched)->toBe('sk_live_4eC39HqLyjWDarjtT1zdp7dc'); + }); +}); + +describe('Offsets are always against the original value', function (): void { + it('keeps a later rule\'s offsets correct after an earlier rule changed the length', function (): void { + config()->set('redactor.profiles.seam', seamProfile([ + 'patterns' => [ + 'email' => SEAM_EMAIL, + 'card' => ['pattern' => '/\b\d{16}\b/', 'validator' => 'luhn'], + ], + 'shannon_entropy' => ['enabled' => false], + ])); + + $line = 'contact a@b.com card 4111111111111111 end'; + $result = resolve(Redactor::class)->inspect($line, 'seam'); + + $byRule = []; + foreach ($result->findings as $finding) { + $byRule[$finding->rule] = $finding->offset; + } + + expect($byRule['email'])->toBe(strpos($line, 'a@b.com')) + ->and($byRule['card'])->toBe(strpos($line, '4111')) + ->and($result->value)->toBe('contact [REDACTED] card [REDACTED] end'); + }); + + it('gives the scanner the right column for the second finding on a line', function (): void { + $path = tempnam(sys_get_temp_dir(), 'seam'); + $line = 'contact a@b.com card 4111111111111111 end'; + file_put_contents($path, $line."\n"); + + try { + $findings = resolve(Scanner::class)->scanFile($path, 'file_scan')->findings; + } finally { + unlink($path); + } + + $columns = []; + foreach ($findings as $finding) { + $columns[$finding->rule] = $finding->column; + } + + expect($columns['email'])->toBe(strpos($line, 'a@b.com') + 1) + ->and($columns['credit_card'])->toBe(strpos($line, '4111') + 1); + }); +}); + +describe('Entropy detections are first-class', function (): void { + it('carries a score and its signals', function (): void { + config()->set('redactor.profiles.seam', seamProfile(['patterns' => []])); + + $result = resolve(Redactor::class)->inspect(['v' => 'Zx7Qm4Kd9Rb2Vn6Tp1Ws8Yc3Hf'], 'seam'); + + expect($result->findings[0]->rule)->toBe('shannon_entropy') + ->and($result->findings[0]->entity)->toBe('high_entropy') + ->and($result->findings[0]->confidence?->score)->toBeGreaterThanOrEqual(0.5) + ->and(implode(' ', $result->findings[0]->confidence?->explain() ?? []))->toContain('entropy'); + }); + + it('scores higher beside a credential keyword', function (): void { + config()->set('redactor.profiles.seam', seamProfile(['patterns' => []])); + + $bare = resolve(Redactor::class)->inspect(['v' => 'Zx7Qm4Kd9Rb2Vn6Tp1Ws8Yc3Hf'], 'seam'); + $labelled = resolve(Redactor::class)->inspect(['v' => 'token=Zx7Qm4Kd9Rb2Vn6Tp1Ws8Yc3Hf'], 'seam'); + + expect($labelled->findings[0]->confidence?->score) + ->toBeGreaterThan($bare->findings[0]->confidence?->score ?? 1.0); + }); + + it('respects the confidence floor', function (): void { + config()->set('redactor.profiles.seam', seamProfile(['patterns' => [], 'min_confidence' => 0.99])); + + $result = resolve(Redactor::class)->inspect(['v' => 'Zx7Qm4Kd9Rb2Vn6Tp1Ws8Yc3Hf'], 'seam'); + + expect($result->wasRedacted)->toBeFalse() + ->and($result->value)->toBe(['v' => 'Zx7Qm4Kd9Rb2Vn6Tp1Ws8Yc3Hf']); + }); + + it('goes through the configured operator', function (): void { + config()->set('redactor.profiles.seam', seamProfile([ + 'patterns' => [], + 'operators' => ['default' => 'hash'], + ])); + + $result = resolve(Redactor::class)->redact(['v' => 'note Zx7Qm4Kd9Rb2Vn6Tp1Ws8Yc3Hf end'], 'seam'); + + expect($result['v'])->toMatch('/^note \[high_entropy:[a-z0-9]+\] end$/'); + }); + + it('still fails closed when the tokeniser cannot split the value', function (): void { + config()->set('redactor.profiles.seam', seamProfile([ + 'patterns' => [], + 'operators' => ['default' => 'hash'], + ])); + + $result = resolve(Redactor::class)->redact(['v' => "\xff\xfe bad Zx7Qm4Kd9Rb2Vn6Tp1Ws8Yc3Hf"], 'seam'); + + // Plain replacement, whatever the operator policy says: there is + // nothing meaningful to hash. + expect($result['v'])->toBe('[REDACTED]'); + }); +}); + +describe('Overlap resolution', function (): void { + it('lets a validated card beat the digit run that also matched it', function (): void { + config()->set('redactor.profiles.seam', seamProfile([ + 'patterns' => [ + 'digits' => ['pattern' => '/\d+/', 'entity' => 'digits'], + 'card' => ['pattern' => '/\b\d{16}\b/', 'validator' => 'luhn', 'entity' => 'credit_card'], + ], + 'operators' => ['default' => 'redact', 'credit_card' => ['partial' => ['keep' => 4]]], + 'shannon_entropy' => ['enabled' => false], + ])); + + expect(resolve(Redactor::class)->redact('paid 4111111111111111 ok', 'seam')) + ->toBe('paid ************1111 ok'); + }); + + it('lets the rule listed first win an equal-score overlap', function (): void { + config()->set('redactor.profiles.seam', seamProfile([ + 'patterns' => [ + 'url_with_auth' => ['pattern' => '/(https?:\/\/[^:\/\s]+:)([^@\/\s]+)(@)/', 'capture' => 2], + 'email' => SEAM_EMAIL, + ], + 'shannon_entropy' => ['enabled' => false], + ])); + + expect(resolve(Redactor::class)->redact('https://admin:hunter2@db.example.com/x', 'seam')) + ->toBe('https://admin:[REDACTED]@db.example.com/x'); + }); + + it('never returns overlapping spans', function (): void { + $kept = DetectionSet::resolve([ + seamDetection('a', 0, 'aaaaa', 0.6), + seamDetection('b', 3, 'bbbbb', 0.7), + seamDetection('c', 6, 'ccccc', 0.65), + seamDetection('d', 20, 'dd', 0.2), + ], 0.3); + + expect(array_map(fn (Detection $d): string => $d->rule, $kept))->toBe(['b']); + }); + + it('keeps the order of arrival as the tie-break, not the order of offset', function (): void { + $kept = DetectionSet::resolve([ + seamDetection('later', 2, 'xxxx'), + seamDetection('earlier', 0, 'yyyy'), + ]); + + expect(array_map(fn (Detection $d): string => $d->rule, $kept))->toBe(['later']); + }); + + it('lets a fail-closed detection swallow everything', function (): void { + $kept = DetectionSet::resolve([ + seamDetection('a', 0, 'aaaaa', 0.9), + Detection::failClosed('x', 'x', 'aaaaa bbbbb', '', 'engine gave up'), + ], 0.95); + + expect($kept)->toHaveCount(1) + ->and($kept[0]->failClosed)->toBeTrue(); + }); +}); + +describe('Preserved detections are reported, not redacted', function (): void { + it('lists the finding without marking the payload redacted', function (): void { + config()->set('redactor.profiles.seam', seamProfile([ + 'operators' => ['default' => 'redact', 'email' => 'preserve'], + 'shannon_entropy' => ['enabled' => false], + ])); + + $result = resolve(Redactor::class)->inspect(['v' => 'hi bob@example.com'], 'seam'); + + expect($result->value)->toBe(['v' => 'hi bob@example.com']) + ->and($result->wasRedacted)->toBeFalse() + ->and($result->findings)->toHaveCount(1) + ->and($result->findings[0]->rule)->toBe('email'); + }); +}); + +describe('Keyword prefilter', function (): void { + it('skips a rule when none of its keywords appear in the subject', function (): void { + config()->set('redactor.profiles.seam', seamProfile([ + 'patterns' => [ + 'phone_bare' => ['pattern' => '/\b\d{10}\b/', 'keywords' => ['phone', 'tel']], + ], + 'shannon_entropy' => ['enabled' => false], + ])); + + expect(resolve(Redactor::class)->redact('started at 1694600000', 'seam')) + ->toBe('started at 1694600000') + ->and(resolve(Redactor::class)->redact('Phone: 5558675309', 'seam')) + ->toBe('Phone: [REDACTED]'); + }); + + it('matches keywords case-insensitively', function (): void { + config()->set('redactor.profiles.seam', seamProfile([ + 'patterns' => [ + 'email' => ['pattern' => SEAM_EMAIL, 'keywords' => ['@']], + ], + 'shannon_entropy' => ['enabled' => false], + ])); + + expect(resolve(Redactor::class)->redact('BOB@EXAMPLE.COM', 'seam'))->toBe('[REDACTED]'); + }); + + it('rejects a non-list keywords option', function (): void { + config()->set('redactor.profiles.seam', seamProfile([ + 'patterns' => ['x' => ['pattern' => '/x/', 'keywords' => 'phone']], + ])); + + resolve(Redactor::class)->redact('x', 'seam'); + })->throws(\InvalidArgumentException::class, 'keywords'); +}); + +describe('Pattern min_length', function (): void { + it('skips a subject shorter than the rule can match, and only then', function (): void { + config()->set('redactor.profiles.seam', seamProfile([ + 'patterns' => ['digits' => ['pattern' => '/\d+/', 'min_length' => 5]], + 'shannon_entropy' => ['enabled' => false], + ])); + + expect(resolve(Redactor::class)->redact('1234', 'seam'))->toBe('1234') + ->and(resolve(Redactor::class)->redact('12345', 'seam'))->toBe('[REDACTED]') + ->and(resolve(Redactor::class)->redact('ab 12', 'seam'))->toBe('ab [REDACTED]'); + }); + + it('rejects a non-positive min_length', function (): void { + config()->set('redactor.profiles.seam', seamProfile([ + 'patterns' => ['digits' => ['pattern' => '/\d+/', 'min_length' => 0]], + ])); + + resolve(Redactor::class)->redact('1', 'seam'); + })->throws(\InvalidArgumentException::class, 'min_length'); + + it('declares no shipped min_length above the length of the secret it catches', function (): void { + // Every planted secret in the shipped-pattern suite must still be + // caught; this pins the cheaper invariant that no rule declares a + // minimum its own sample would fail. + foreach (RedactorConfig::fromConfig('default')->patterns as $rule) { + expect($rule->minLength)->toBeGreaterThanOrEqual(1); + } + + $probe = [ + 'aws_access_key' => 'AKIAIOSFODNN7EXAMPLE', + 'stripe_key' => 'sk_live_4eC39HqLyjWDarjtT1zdp7dc', + 'slack_token' => 'xoxb-1234567890-abcdefghijABCDEFGHIJ', + 'email' => 'a@b.c', + 'url_with_auth' => 'a://:x@', + 'ssn' => '123-45-6789', + 'credit_card' => '4111111111111', + ]; + + foreach ($probe as $rule => $shortest) { + $config = RedactorConfig::fromConfig('default'); + expect($config->patterns[$rule]->minLength)->toBeLessThanOrEqual(strlen($shortest), $rule); + } + }); +}); + +/** + * A chainable strategy that records the string it was handed. + */ +class SeamWitnessStrategy implements ChainableStrategy, Strategy +{ + public static ?string $seen = null; + + public function shouldHandle(mixed $value, string $key, RedactionContext $context): bool + { + return is_string($value); + } + + public function handle(mixed $value, string $key, RedactionContext $context): mixed + { + self::$seen = is_string($value) ? $value : null; + + return $value.' !'; + } +} + +describe('The seam between detectors and everything after them', function (): void { + beforeEach(function (): void { + SeamWitnessStrategy::$seen = null; + config()->set('redactor.profiles.seam', seamProfile()); + }); + + it('acts on the detections before a non-detecting strategy gets to see the value', function (): void { + $redactor = resolve(Redactor::class); + $redactor->registerCustomStrategy('seam_witness', new SeamWitnessStrategy); + config()->set('redactor.profiles.seam.strategies', [RegexPatternsStrategy::class, 'seam_witness']); + + expect($redactor->redact('mail bob@example.com', 'seam'))->toBe('mail [REDACTED] !') + ->and(SeamWitnessStrategy::$seen)->toBe('mail [REDACTED]'); + }); + + it('skips a detection whose offset lies before the cursor rather than splice garbage', function (): void { + $context = new RedactionContext(RedactorConfig::fromConfig('seam')); + $context->collect(seamDetection('email', -1, 'bob')); + $context->collect(seamDetection('email', 5, 'bob')); + + expect($context->resolvePendingDetections('mail bob now', 'k'))->toBe('mail [REDACTED] now') + ->and($context->getFindings())->toHaveCount(1); + }); + + it('uses the profile secrets alone when none were registered at runtime', function (): void { + $config = RedactorConfig::fromConfig('seam'); + + expect((new RedactionContext($config))->secrets())->toBe($config->knownSecrets); + }); + + it('hands anything but a string back untouched from every detecting strategy', function (): void { + $context = new RedactionContext(RedactorConfig::fromConfig('seam')); + $payload = ['nested' => 'bob@example.com']; + + foreach ([new KnownSecretsStrategy, new RegexPatternsStrategy, new ShannonEntropyStrategy, new EntityRecognitionStrategy] as $strategy) { + expect($strategy->handle($payload, 'k', $context))->toBe($payload) + ->and($context->hasPendingDetections())->toBeFalse(); + } + }); + + it('reports nothing from the entropy detector for a subject shorter than min_length', function (): void { + $context = new RedactionContext(RedactorConfig::fromConfig('seam')); + + expect((new ShannonEntropyStrategy)->detect('Zx7Qm4Kd9Rb', 'k', $context))->toBe([]); + }); +}); diff --git a/tests/Feature/RedactorDispatchTest.php b/tests/Feature/RedactorDispatchTest.php new file mode 100644 index 0000000..250ba2b --- /dev/null +++ b/tests/Feature/RedactorDispatchTest.php @@ -0,0 +1,123 @@ + */ + public static array $keys = []; + + public static function reset(): void + { + self::$keys = []; + } + + public function shouldHandle(mixed $value, string $key, RedactionContext $context): bool + { + self::$keys[] = $key; + + return false; + } + + public function handle(mixed $value, string $key, RedactionContext $context): mixed + { + return $value; + } +} + +function dispatchProfile(array $overrides = []): array +{ + return array_merge([ + 'enabled' => true, + 'strategies' => ['counting'], + 'safe_keys' => [], + 'blocked_keys' => [], + 'patterns' => [], + 'replacement' => '[REDACTED]', + 'mark_redacted' => false, + 'track_redacted_keys' => false, + 'non_redactable_object_behavior' => 'preserve', + 'max_value_length' => null, + 'redact_large_objects' => false, + 'max_object_size' => 100, + 'shannon_entropy' => ['enabled' => false], + ], $overrides); +} + +describe('Strategy dispatch', function (): void { + beforeEach(function (): void { + CountingStrategy::reset(); + config()->set('redactor.custom_strategies', ['counting' => CountingStrategy::class]); + config()->set('redactor.profiles.dispatch', dispatchProfile()); + }); + + it('evaluates each node exactly once', function (): void { + // {a: {b: {c: 1}}} is four nodes: the root, a, b and c. redactArray() + // used to re-run the chain on every nested array with an empty key, + // after the parent loop had already run it with the real key - six + // dispatches for four nodes. + resolve(Redactor::class)->redact(['a' => ['b' => ['c' => 1]]], 'dispatch'); + + expect(CountingStrategy::$keys)->toBe(['', 'a', 'b', 'c']); + }); + + it('evaluates a wider tree once per node', function (): void { + resolve(Redactor::class)->redact([ + 'x' => ['p' => 1, 'q' => 2], + 'y' => ['r' => ['s' => 3]], + ], 'dispatch'); + + // root, x, p, q, y, r, s + expect(CountingStrategy::$keys)->toHaveCount(7); + }); + + it('evaluates the root once for a top-level array', function (): void { + resolve(Redactor::class)->redact(['only' => 'value'], 'dispatch'); + + expect(CountingStrategy::$keys)->toBe(['', 'only']); + }); + + it('still evaluates the root array as a whole so LargeObjectStrategy applies', function (): void { + config()->set('redactor.profiles.dispatch_large', dispatchProfile([ + 'strategies' => [LargeObjectStrategy::class], + 'redact_large_objects' => true, + 'max_object_size' => 3, + ])); + + $result = resolve(Redactor::class)->redact(['a' => 1, 'b' => 2, 'c' => 3, 'd' => 4], 'dispatch_large'); + + expect($result)->toHaveKey('_large_object_redacted'); + }); + + it('still evaluates a nested array as a whole so LargeObjectStrategy applies', function (): void { + config()->set('redactor.profiles.dispatch_large', dispatchProfile([ + 'strategies' => [LargeObjectStrategy::class], + 'redact_large_objects' => true, + 'max_object_size' => 3, + ])); + + $result = resolve(Redactor::class)->redact([ + 'small' => ['a' => 1], + 'big' => ['a' => 1, 'b' => 2, 'c' => 3, 'd' => 4], + ], 'dispatch_large'); + + expect($result['big'])->toHaveKey('_large_object_redacted') + ->and($result['small'])->toBe(['a' => 1]); + }); + + it('does not skip the chain for a scalar', function (): void { + resolve(Redactor::class)->redact('a bare string', 'dispatch'); + + expect(CountingStrategy::$keys)->toBe(['']); + }); +}); diff --git a/tests/Feature/RedactorEntityFilterTest.php b/tests/Feature/RedactorEntityFilterTest.php new file mode 100644 index 0000000..5441992 --- /dev/null +++ b/tests/Feature/RedactorEntityFilterTest.php @@ -0,0 +1,86 @@ + 'bob@example.com', + 'password' => 'hunter2', + 'note' => 'card 4111111111111111 for alice@example.com, token sk_live_4eC39HqLyjWDarjtT1zdp7dc', + ]; +} + +describe('Per-call entity filtering', function (): void { + it('acts only on the entities asked for', function (): void { + $result = Redactor::profile('default')->only(['email'])->withoutMarkers()->redact(entityFilterPayload()); + + expect($result)->toBe([ + 'email' => '[REDACTED]', + 'password' => 'hunter2', + 'note' => 'card 4111111111111111 for [REDACTED], token sk_live_4eC39HqLyjWDarjtT1zdp7dc', + ]); + }); + + it('skips the entities excluded', function (): void { + $result = Redactor::profile('default')->except(['email', 'credit_card'])->withoutMarkers()->redact(entityFilterPayload()); + + expect($result['email'])->toBe('bob@example.com') + ->and($result['password'])->toBe('[REDACTED]') + ->and($result['note'])->toBe('card 4111111111111111 for alice@example.com, token [REDACTED]'); + }); + + it('treats a key rule\'s entity as the key name', function (): void { + $result = Redactor::profile('default')->only(['password'])->withoutMarkers()->redact(entityFilterPayload()); + + expect($result['password'])->toBe('[REDACTED]') + ->and($result['email'])->toBe('bob@example.com'); + }); + + it('applies to path rules by the key they land on', function (): void { + config()->set('redactor.profiles.default.paths', ['meta.token' => 'redact', 'meta.note' => 'redact']); + + $result = Redactor::profile('default')->only(['note'])->withoutMarkers() + ->redact(['meta' => ['token' => 'abc', 'note' => 'n']]); + + expect($result['meta'])->toBe(['token' => 'abc', 'note' => '[REDACTED]']); + }); + + it('reports only the findings it acted on', function (): void { + $result = Redactor::profile('default')->only(['credit_card'])->inspect(entityFilterPayload()); + + expect(array_unique(array_map(fn (MatchFinding $f): string => $f->entity(), $result->findings)))->toBe(['credit_card']) + ->and($result->redactedKeys)->toBe(['note']); + }); + + it('compares entities case-insensitively and composes only with except', function (): void { + $filter = EntityFilter::all()->only(['Email', 'CREDIT_CARD'])->except(['credit_card']); + + expect($filter->allows('email'))->toBeTrue() + ->and($filter->allows('credit_card'))->toBeFalse() + ->and($filter->allows('password'))->toBeFalse() + ->and($filter->allowsEverything())->toBeFalse() + ->and(EntityFilter::all()->allowsEverything())->toBeTrue(); + }); + + it('never lets a filter suppress a fail-closed detection', function (): void { + config()->set('redactor.profiles.default.patterns', ['bad' => '/^\p{L}+$/u']); + + expect(Redactor::profile('default')->only(['nothing'])->redact("\xff\xfe"))->toBe('[REDACTED]'); + }); + + it('is recorded by the fake', function (): void { + $fake = Redactor::fake(); + + Redactor::profile('default')->only(['email'])->redact(entityFilterPayload()); + + $fake->assertRedacted('email'); + $fake->assertNotRedacted('password'); + }); +}); diff --git a/tests/Feature/RedactorEntityRecognitionTest.php b/tests/Feature/RedactorEntityRecognitionTest.php new file mode 100644 index 0000000..e94143f --- /dev/null +++ b/tests/Feature/RedactorEntityRecognitionTest.php @@ -0,0 +1,673 @@ + true, + 'strategies' => [RegexPatternsStrategy::class, EntityRecognitionStrategy::class], + 'safe_keys' => [], + 'blocked_keys' => [], + 'patterns' => ['email' => ['pattern' => '/[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+/', 'entity' => 'email']], + 'operators' => ['default' => 'redact'], + 'recognition' => [ + 'enabled' => true, + 'driver' => 'presidio', + 'url' => NER_URL, + 'language' => 'en', + 'entities' => ['PERSON', 'LOCATION'], + 'entity_map' => ['PERSON' => 'person', 'LOCATION' => 'location'], + 'score_threshold' => 0.6, + 'min_length' => 10, + 'max_length' => 5000, + 'min_words' => 3, + 'timeout' => 1, + 'failure_threshold' => 2, + 'cooldown' => 60, + ], + 'replacement' => '[REDACTED]', + 'mark_redacted' => false, + 'track_redacted_keys' => false, + 'non_redactable_object_behavior' => 'preserve', + 'max_value_length' => null, + 'redact_large_objects' => false, + 'max_object_size' => 100, + 'pseudonymization' => ['key' => testPseudonymizationKey()], + 'shannon_entropy' => ['enabled' => false], + ], $overrides); +} + +/** Presidio-shaped response for spans given as [entity, start, end, score]. */ +function presidio(array $spans): array +{ + return array_map(fn (array $s): array => ['entity_type' => $s[0], 'start' => $s[1], 'end' => $s[2], 'score' => $s[3]], $spans); +} + +describe('Entity recognition', function (): void { + beforeEach(function (): void { + CircuitBreaker::reset(); + config()->set('redactor.profiles.ner', nerProfile()); + }); + + it('is off unless the profile enables it', function (): void { + Http::fake(); + config()->set('redactor.profiles.ner.recognition.enabled', false); + + expect(resolve(Redactor::class)->redact('Please call John Smith about the invoice', 'ner')) + ->toBe('Please call John Smith about the invoice'); + + Http::assertNothingSent(); + }); + + it('redacts a recognised person and goes through the entity operator', function (): void { + Http::fake([NER_URL => Http::response(presidio([['PERSON', 12, 22, 0.85]]))]); + config()->set('redactor.profiles.ner.operators', ['default' => 'redact', 'person' => 'hash']); + + $text = 'Please call John Smith about the invoice'; + + expect(resolve(Redactor::class)->redact($text, 'ner')) + ->toMatch('/^Please call \[person:[a-z0-9]+\] about the invoice$/'); + }); + + it('sends the text, language, entities and threshold Presidio expects', function (): void { + Http::fake([NER_URL => Http::response([])]); + + resolve(Redactor::class)->redact('Please call John Smith about the invoice', 'ner'); + + Http::assertSent(fn ($request): bool => $request->url() === NER_URL + && $request['text'] === 'Please call John Smith about the invoice' + && $request['language'] === 'en' + && $request['entities'] === ['PERSON', 'LOCATION'] + && $request['score_threshold'] === 0.6); + }); + + it('converts character offsets to bytes correctly after multibyte text', function (): void { + // "Café " is 5 characters and 6 bytes; the name starts at character 5. + $text = 'Café with Jürgen Müller yesterday'; + Http::fake([NER_URL => Http::response(presidio([['PERSON', 10, 23, 0.9]]))]); + + $result = resolve(Redactor::class)->inspect($text, 'ner'); + + expect($result->value)->toBe('Café with [REDACTED] yesterday') + ->and($result->findings[0]->matched)->toBe('Jürgen Müller') + ->and($result->findings[0]->offset)->toBe(strlen('Café with ')); + }); + + it('skips a span whose offsets do not land on the subject', function (): void { + // Asked per value, since the batch driver already drops spans outside a text... + config()->set('redactor.profiles.ner.recognition.batch', false); + Http::fake([NER_URL => Http::response(presidio([['PERSON', 30, 60, 0.9], ['PERSON', 12, 22, 0.9]]))]); + + $text = 'Please call John Smith about the invoice'; + + expect(resolve(Redactor::class)->redact($text, 'ner'))->toBe('Please call [REDACTED] about the invoice'); + }); + + it('ignores labels the profile did not ask for and scores under the threshold', function (): void { + Http::fake([NER_URL => Http::response(presidio([ + ['ORGANIZATION', 0, 6, 0.95], + ['PERSON', 12, 22, 0.4], + ['LOCATION', 33, 40, 0.7], + ]))]); + + $text = 'Please call John Smith about the invoice'; + + expect(resolve(Redactor::class)->redact($text, 'ner'))->toBe('Please call John Smith about the [REDACTED]'); + }); + + it('works alongside the pattern detectors in one rewrite', function (): void { + Http::fake([NER_URL => Http::response(presidio([['PERSON', 0, 10, 0.9]]))]); + + expect(resolve(Redactor::class)->redact('John Smith wrote to bob@example.com today', 'ner')) + ->toBe('[REDACTED] wrote to [REDACTED] today'); + }); + + it('does not ask the model about values that are not prose', function (): void { + Http::fake(); + + resolve(Redactor::class)->redact(['json' => '{"name":"John Smith","note":"call him"}', 'token' => 'Zx7Qm4Kd9Rb2Vn6Tp1Ws8Yc3Hf', 'short' => 'hi'], 'ner'); + + Http::assertNothingSent(); + }); + + it('never throws when the recogniser fails, and trips the breaker after repeated failures', function (): void { + Http::fake([NER_URL => Http::response('down', 503)]); + $text = 'Please call John Smith about the invoice'; + + $redactor = resolve(Redactor::class); + + expect($redactor->redact($text, 'ner'))->toBe($text) + ->and($redactor->redact($text, 'ner'))->toBe($text) + ->and($redactor->redact($text, 'ner'))->toBe($text); + + // Threshold is 2: the third call is skipped without a request. + Http::assertSentCount(2); + expect(CircuitBreaker::isOpen('presidio|ner'))->toBeTrue(); + }); + + it('closes the breaker again on success', function (): void { + CircuitBreaker::recordFailure('presidio|ner', 1, 0); + CircuitBreaker::recordSuccess('presidio|ner'); + + expect(CircuitBreaker::allows('presidio|ner'))->toBeTrue(); + }); + + it('accepts a recogniser registered at runtime', function (): void { + $redactor = resolve(Redactor::class); + $redactor->registerRecognizer(new class implements Recognizer + { + public function name(): string + { + return 'stub'; + } + + public function recognize(string $text, string $language, array $entities, float $scoreThreshold): array + { + return [new RecognizedSpan('PERSON', 12, 22, 0.99)]; + } + }); + config()->set('redactor.profiles.ner.recognition.driver', 'stub'); + + expect($redactor->redact('Please call John Smith about the invoice', 'ner')) + ->toBe('Please call [REDACTED] about the invoice'); + }); + + it('falls back to rules only for an unknown driver', function (): void { + Http::fake(); + config()->set('redactor.profiles.ner.recognition.driver', 'nope'); + + expect(resolve(Redactor::class)->redact('John Smith wrote to bob@example.com today', 'ner')) + ->toBe('John Smith wrote to [REDACTED] today'); + }); + + it('reports the recogniser and score in the finding', function (): void { + Http::fake([NER_URL => Http::response(presidio([['PERSON', 12, 22, 0.85]]))]); + + $result = resolve(Redactor::class)->inspect('Please call John Smith about the invoice', 'ner'); + + expect($result->findings[0]->rule)->toBe('entity_recognition') + ->and($result->findings[0]->entity)->toBe('person') + ->and($result->findings[0]->confidence?->score)->toBe(0.85) + ->and(implode(' ', $result->findings[0]->confidence?->explain() ?? []))->toContain('presidio'); + }); +}); + +describe('Conditional strategies', function (): void { + it('leaves a disabled recognition strategy out of the chain and brings it back when enabled', function (): void { + config()->set('redactor.profiles.ner', nerProfile(['recognition' => ['enabled' => false]])); + $redactor = resolve(Redactor::class); + + $classes = fn (): array => array_map(fn (Strategy $s): string => $s::class, $redactor->strategies('ner')); + + expect($classes())->not->toContain(EntityRecognitionStrategy::class); + + config()->set('redactor.profiles.ner.recognition', nerProfile()['recognition']); + + expect($classes())->toContain(EntityRecognitionStrategy::class); + }); + + it('does not report a disabled strategy as unresolvable', function (): void { + config()->set('redactor.profiles.ner', nerProfile(['recognition' => ['enabled' => false]])); + + expect(resolve(Redactor::class)->validateProfiles())->not->toHaveKey('ner'); + }); +}); + +describe('Entity recognition at its edges', function (): void { + beforeEach(function (): void { + CircuitBreaker::reset(); + config()->set('redactor.profiles.ner', nerProfile()); + }); + + it('skips a span that covers only whitespace', function (): void { + Http::fake([NER_URL => Http::response(presidio([['PERSON', 11, 12, 0.9]]))]); + $text = 'Please call John Smith about the invoice'; + + $result = resolve(Redactor::class)->inspect($text, 'ner'); + + expect($result->value)->toBe($text) + ->and($result->findings)->toBe([]); + }); + + it('maps a label through entity_map, and falls back to the lowercased label when the map is not a map', function (): void { + Http::fake([NER_URL => Http::response(presidio([['PERSON', 12, 22, 0.85]]))]); + $text = 'Please call John Smith about the invoice'; + + config()->set('redactor.profiles.ner.recognition.entity_map', ['PERSON' => 'customer']); + $mapped = resolve(Redactor::class)->inspect($text, 'ner')->findings[0]->entity; + + config()->set('redactor.profiles.ner.recognition.entity_map', 'customer'); + $unmapped = resolve(Redactor::class)->inspect($text, 'ner')->findings[0]->entity; + + expect($mapped)->toBe('customer') + ->and($unmapped)->toBe('person'); + }); + + it('declines prose when the profile has recognition switched off, even when asked directly', function (): void { + config()->set('redactor.profiles.ner.recognition.enabled', false); + $context = new RedactionContext(RedactorConfig::fromConfig('ner')); + + expect((new EntityRecognitionStrategy)->shouldHandle('Please call John Smith about the invoice', 'k', $context))->toBeFalse(); + }); + + it('rejects a Presidio body that is not a list', function (): void { + Http::fake([NER_URL => Http::response('"just a string"', 200, ['Content-Type' => 'application/json'])]); + + expect(fn (): array => (new PresidioRecognizer(NER_URL, 1.0))->recognize('Please call John Smith', 'en', [], 0.5)) + ->toThrow(RuntimeException::class, 'non-list'); + }); + + it('skips malformed Presidio items and keeps the well-formed ones', function (): void { + Http::fake([NER_URL => Http::response([ + 'nope', + ['entity_type' => 'PERSON', 'start' => 'x', 'end' => 4, 'score' => 0.9], + ['entity_type' => 'PERSON', 'start' => 12, 'end' => 22, 'score' => 0.9], + ])]); + + $spans = (new PresidioRecognizer(NER_URL, 1.0))->recognize('Please call John Smith', 'en', ['PERSON'], 0.5); + + expect($spans)->toHaveCount(1) + ->and($spans[0]->start)->toBe(12) + ->and($spans[0]->end)->toBe(22); + }); + + it('exposes the recogniser registry with the built-in and anything registered since', function (): void { + $redactor = resolve(Redactor::class); + + expect($redactor->recognizers()->has('presidio'))->toBeTrue() + ->and($redactor->recognizers()->has('stub'))->toBeFalse(); + + $redactor->registerRecognizer(new class implements Recognizer + { + public function name(): string + { + return 'stub'; + } + + public function recognize(string $text, string $language, array $entities, float $scoreThreshold): array + { + return []; + } + }); + + expect($redactor->recognizers()->has('stub'))->toBeTrue(); + }); +}); + +describe('Entity recognition batching', function (): void { + beforeEach(function (): void { + CircuitBreaker::reset(); + config()->set('redactor.profiles.ner', nerProfile()); + }); + + /** Answer one joined request by locating each name in the joined text. */ + function presidioFinding(array $names): callable + { + return function ($request) use ($names): PromiseInterface { + $text = $request->data()['text']; + $spans = []; + + foreach ($names as [$entity, $name, $score]) { + $start = mb_strpos($text, $name); + + if ($start !== false) { + $spans[] = [$entity, $start, $start + mb_strlen($name), $score]; + } + } + + return Http::response(presidio($spans)); + }; + } + + it('sends every prose value in one request and maps each span back to its own value', function (): void { + Http::fake([NER_URL => presidioFinding([['PERSON', 'John Smith', 0.9], ['PERSON', 'Zoë Müller', 0.9], ['LOCATION', 'Berlin', 0.8]])]); + + $payload = [ + 'notes' => 'Please call John Smith about the invoice', + 'nested' => [ + 'summary' => 'Café visit with Zoë Müller went well today', + 'city' => 'She is now based in Berlin for the year', + ], + 'count' => 3, + 'short' => 'no', + ]; + + $result = resolve(Redactor::class)->inspect($payload, 'ner'); + + expect($result->value['notes'])->toBe('Please call [REDACTED] about the invoice') + ->and($result->value['nested']['summary'])->toBe('Café visit with [REDACTED] went well today') + ->and($result->value['nested']['city'])->toBe('She is now based in [REDACTED] for the year') + ->and($result->value['count'])->toBe(3); + + Http::assertSentCount(1); + Http::assertSent(fn ($request): bool => str_contains($request->data()['text'], "invoice\n\nCafé")); + }); + + it('sends identical texts once and leaves safe and blocked keys out of the batch', function (): void { + config()->set('redactor.profiles.ner.safe_keys', ['public_note', 'public_notes']); + config()->set('redactor.profiles.ner.blocked_keys', ['secret_note']); + config()->set('redactor.profiles.ner.strategies', [ + SafeKeysStrategy::class, + BlockedKeysStrategy::class, + RegexPatternsStrategy::class, + EntityRecognitionStrategy::class, + ]); + Http::fake([NER_URL => presidioFinding([['PERSON', 'John Smith', 0.9]])]); + + $payload = [ + 'a' => 'Please call John Smith about the invoice', + 'b' => 'Please call John Smith about the invoice', + 'public_note' => 'Alice Jones wrote this public note for everyone', + 'secret_note' => 'Bob Brown wrote this private note for nobody', + 'public_notes' => ['Alice Jones wrote this public note for everyone too'], + ]; + + $result = resolve(Redactor::class)->inspect($payload, 'ner'); + + expect($result->value['a'])->toBe('Please call [REDACTED] about the invoice') + ->and($result->value['b'])->toBe('Please call [REDACTED] about the invoice') + ->and($result->value['public_note'])->toBe('Alice Jones wrote this public note for everyone') + ->and($result->value['secret_note'])->toBe('[REDACTED]'); + + Http::assertSentCount(1); + Http::assertSent(function ($request): bool { + $text = $request->data()['text']; + + return substr_count($text, 'John Smith') === 1 + && ! str_contains($text, 'Alice Jones') + && ! str_contains($text, 'Bob Brown'); + }); + }); + + it('gathers prose from objects the way the walk opens them', function (): void { + Http::fake([NER_URL => presidioFinding([['PERSON', 'John Smith', 0.9], ['PERSON', 'Jane Doe', 0.9]])]); + + $arrayable = new class implements Arrayable + { + public function toArray(): array + { + return ['note' => 'Please call John Smith about the invoice']; + } + }; + + $plain = new \stdClass; + $plain->note = 'Please call Jane Doe about the refund'; + + // JSON cannot encode a self-reference, so this one is skipped, as the walk skips it... + $unopenable = new \stdClass; + $unopenable->self = $unopenable; + $unopenable->note = 'Please call Bob Brown about the delivery'; + + $payload = [ + 'model' => $arrayable, + 'plain' => $plain, + 'unopenable' => $unopenable, + 'exception' => new RuntimeException('Please call John Smith about the invoice'), + 'when' => new \DateTimeImmutable('2024-01-01'), + ]; + + $result = resolve(Redactor::class)->inspect($payload, 'ner'); + + expect($result->value['model']['note'])->toBe('Please call [REDACTED] about the invoice') + ->and($result->value['plain']['note'])->toBe('Please call [REDACTED] about the refund'); + + Http::assertSentCount(1); + Http::assertSent(fn ($request): bool => ! str_contains($request->data()['text'], 'Bob Brown')); + }); + + it('stops gathering at the depth limit and on a cycle, like the walk', function (): void { + Http::fake([NER_URL => presidioFinding([['PERSON', 'John Smith', 0.9], ['PERSON', 'Jane Smith', 0.9]])]); + + $cyclic = new class implements Arrayable + { + public function toArray(): array + { + return ['self' => $this, 'note' => 'Please call Jane Smith about the refund']; + } + }; + + $payload = ['one' => ['two' => ['three' => 'Please call John Smith about the invoice']], 'cyclic' => $cyclic]; + + // Deep enough for everything: the cycle is entered once and the deep value is found... + resolve(Redactor::class)->inspect($payload, 'ner'); + + Http::assertSent(fn ($request): bool => substr_count($request->data()['text'], 'Jane Smith') === 1 + && substr_count($request->data()['text'], 'John Smith') === 1); + + // Two levels: the deep value is never gathered... + config()->set('redactor.profiles.ner.max_depth', 2); + resolve(Redactor::class)->inspect($payload, 'ner'); + + Http::assertSent(fn ($request): bool => ! str_contains($request->data()['text'], 'John Smith')); + }); + + it('counts a failed batch once and does not retry per value', function (): void { + Http::fake([NER_URL => Http::response('down', 503)]); + + $payload = [ + 'a' => 'Please call John Smith about the invoice', + 'b' => 'Please call Jane Doe about the refund', + 'c' => 'Please call Bob Brown about the delivery', + ]; + + $result = resolve(Redactor::class)->inspect($payload, 'ner'); + + expect($result->value)->toBe($payload) + ->and(CircuitBreaker::isOpen('presidio|ner'))->toBeFalse(); + + Http::assertSentCount(1); + }); + + it('asks nothing while the breaker is open', function (): void { + Http::fake([NER_URL => Http::response('down', 503)]); + + resolve(Redactor::class)->inspect(['a' => 'Please call John Smith about the invoice'], 'ner'); + resolve(Redactor::class)->inspect(['a' => 'Please call John Smith about the invoice'], 'ner'); + + expect(CircuitBreaker::isOpen('presidio|ner'))->toBeTrue(); + + resolve(Redactor::class)->inspect(['a' => 'Please call John Smith about the invoice'], 'ner'); + + Http::assertSentCount(2); + }); + + it('counts a per-value failure against the breaker when batch is off', function (): void { + config()->set('redactor.profiles.ner.recognition.batch', false); + Http::fake([NER_URL => Http::response('down', 503)]); + + $result = resolve(Redactor::class)->inspect([ + 'a' => 'Please call John Smith about the invoice', + 'b' => 'Please call John Smith about the refund', + ], 'ner'); + + expect($result->value['a'])->toBe('Please call John Smith about the invoice') + ->and(CircuitBreaker::isOpen('presidio|ner'))->toBeTrue(); + + Http::assertSentCount(2); + }); + + it('asks per value when batch is off', function (): void { + config()->set('redactor.profiles.ner.recognition.batch', false); + Http::fake([NER_URL => presidioFinding([['PERSON', 'John Smith', 0.9]])]); + + $result = resolve(Redactor::class)->inspect([ + 'a' => 'Please call John Smith about the invoice', + 'b' => 'Please call John Smith about the refund', + ], 'ner'); + + expect($result->value['a'])->toBe('Please call [REDACTED] about the invoice') + ->and($result->value['b'])->toBe('Please call [REDACTED] about the refund'); + + Http::assertSentCount(2); + }); + + it('falls back to one call for a value the walk truncated before the strategy saw it', function (): void { + config()->set('redactor.profiles.ner.max_value_length', 45); + config()->set('redactor.profiles.ner.strategies', [ + LargeStringStrategy::class, + RegexPatternsStrategy::class, + EntityRecognitionStrategy::class, + ]); + Http::fake([NER_URL => presidioFinding([['PERSON', 'John Smith', 0.9]])]); + + $result = resolve(Redactor::class)->inspect([ + 'a' => 'Please call John Smith about the invoice, the refund and the delivery schedule', + ], 'ner'); + + expect($result->value['a'])->toStartWith('Please call [REDACTED] about the invoice'); + + Http::assertSentCount(2); + }); + + it('asks a recogniser that cannot take a list once per text, before the walk', function (): void { + $redactor = resolve(Redactor::class); + $calls = []; + + $redactor->registerRecognizer(new class($calls) implements Recognizer + { + public function __construct(private array &$calls) {} + + public function name(): string + { + return 'single'; + } + + public function recognize(string $text, string $language, array $entities, float $scoreThreshold): array + { + $this->calls[] = $text; + + return [new RecognizedSpan('PERSON', 12, 22, 0.9)]; + } + }); + config()->set('redactor.profiles.ner.recognition.driver', 'single'); + + $result = $redactor->inspect([ + 'a' => 'Please call John Smith about the invoice', + 'b' => 'Please call Jane Smith about the refund', + ], 'ner'); + + expect($calls)->toBe(['Please call John Smith about the invoice', 'Please call Jane Smith about the refund']) + ->and($result->value['a'])->toBe('Please call [REDACTED] about the invoice') + ->and($result->value['b'])->toBe('Please call [REDACTED] about the refund'); + }); + + it('skips the batch for an unknown driver and for a payload with no prose', function (): void { + Http::fake(); + + config()->set('redactor.profiles.ner.recognition.driver', 'missing'); + resolve(Redactor::class)->inspect(['a' => 'Please call John Smith about the invoice'], 'ner'); + + config()->set('redactor.profiles.ner.recognition.driver', 'presidio'); + resolve(Redactor::class)->inspect(['a' => 'x', 'b' => 42], 'ner'); + + Http::assertNothingSent(); + }); +}); + +describe('PresidioRecognizer::recognizeMany', function (): void { + it('drops a span that crosses the join between two texts', function (): void { + // "Alice" ends text one; "Smith" starts text two; a span over both is an artefact... + Http::fake([NER_URL => Http::sequence() + ->push(presidio([['PERSON', 17, 29, 0.9]])) + ->push(presidio([['PERSON', 17, 22, 0.9], ['PERSON', 24, 29, 0.9]]))]); + + $recognizer = new PresidioRecognizer(NER_URL); + $spans = $recognizer->recognizeMany(['Please say hi to Alice', 'Smith is here now'], 'en', [], 0.5); + + expect($spans[0])->toHaveCount(0) + ->and($spans[1])->toHaveCount(0); + + $spans = $recognizer->recognizeMany(['Please say hi to Alice', 'Smith is here now'], 'en', [], 0.5); + + expect($spans[0][0]->start)->toBe(17) + ->and($spans[0][0]->end)->toBe(22) + ->and($spans[1][0]->start)->toBe(0) + ->and($spans[1][0]->end)->toBe(5); + }); + + it('splits texts into requests under the character cap', function (): void { + Http::fake([NER_URL => Http::response([])]); + + $long = str_repeat('word ', 8000); // 40,000 characters + $spans = (new PresidioRecognizer(NER_URL))->recognizeMany([$long, $long, 'short one here'], 'en', [], 0.5); + + expect($spans)->toHaveCount(3); + + Http::assertSentCount(2); + Http::assertSent(fn ($request): bool => mb_strlen($request->data()['text']) <= PresidioRecognizer::BATCH_CHARACTERS); + }); + + it('keeps input indexes and returns an empty list for every text when nothing is found', function (): void { + Http::fake([NER_URL => Http::response([])]); + + $spans = (new PresidioRecognizer(NER_URL))->recognizeMany([5 => 'Please say hi to Alice', 9 => 'Smith is here now'], 'en', [], 0.5); + + expect($spans)->toBe([5 => [], 9 => []]); + }); + + it('sends nothing for an empty list', function (): void { + Http::fake(); + + expect((new PresidioRecognizer(NER_URL))->recognizeMany([], 'en', [], 0.5))->toBe([]); + + Http::assertNothingSent(); + }); +}); + +describe('Priming strategies', function (): void { + it('see the whole payload once before the walk', function (): void { + $seen = []; + + $strategy = new class($seen) implements PrimingStrategy + { + public function __construct(private array &$seen) {} + + public function prime(mixed $content, RedactionContext $context): void + { + $this->seen[] = $content; + } + + public function shouldHandle(mixed $value, string $key, RedactionContext $context): bool + { + return false; + } + + public function handle(mixed $value, string $key, RedactionContext $context): mixed + { + return $value; + } + }; + + $redactor = resolve(Redactor::class); + $redactor->registerCustomStrategy('primer', $strategy); + config()->set('redactor.profiles.ner', nerProfile(['strategies' => ['primer']])); + + $redactor->redact(['a' => 1, 'b' => ['c' => 2]], 'ner'); + + expect($seen)->toBe([['a' => 1, 'b' => ['c' => 2]]]); + }); +}); diff --git a/tests/Feature/RedactorEnvConfigTest.php b/tests/Feature/RedactorEnvConfigTest.php new file mode 100644 index 0000000..55b1815 --- /dev/null +++ b/tests/Feature/RedactorEnvConfigTest.php @@ -0,0 +1,197 @@ + 'true', + 'strategies' => [LargeObjectStrategy::class], + 'safe_keys' => [], + 'blocked_keys' => [], + 'patterns' => [], + 'replacement' => '[MASKED]', + 'mark_redacted' => 'false', + 'track_redacted_keys' => 'true', + 'non_redactable_object_behavior' => 'redact', + 'max_value_length' => '2500', + 'redact_large_objects' => 'true', + 'max_object_size' => '25', + 'shannon_entropy' => [ + 'enabled' => 'true', + 'threshold' => '4.2', + 'min_length' => '18', + 'exclusion_patterns' => [], + ], + ], $overrides); + } + + it('applies every profile value when it arrives as a string', function (): void { + config()->set('redactor.profiles.env_shaped', envShapedProfile()); + + $config = RedactorConfig::fromConfig('env_shaped'); + + expect($config->enabled)->toBeTrue() + ->and($config->replacement)->toBe('[MASKED]') + ->and($config->markRedacted)->toBeFalse() + ->and($config->trackRedactedKeys)->toBeTrue() + ->and($config->nonRedactableObjectBehavior)->toBe('redact') + ->and($config->maxValueLength)->toBe(2500) + ->and($config->redactLargeObjects)->toBeTrue() + ->and($config->maxObjectSize)->toBe(25) + ->and($config->shannonEntropy['enabled'])->toBeTrue() + ->and($config->shannonEntropy['threshold'])->toBe(4.2) + ->and($config->shannonEntropy['min_length'])->toBe(18); + }); + + it('honours REDACTOR_MAX_OBJECT_SIZE end to end', function (): void { + // is_int() rejected the string form, so this knob always fell back to + // 100 and arrays of 26-100 items were never redacted. + config()->set('redactor.profiles.env_shaped', envShapedProfile([ + 'mark_redacted' => 'true', + ])); + + $payload = array_fill_keys( + array_map(fn (int $i): string => "field_{$i}", range(1, 30)), + 'value' + ); + + $result = resolve(Redactor::class)->redact($payload, 'env_shaped'); + + expect($result)->toHaveKey('_large_object_redacted'); + }); + + it('honours REDACTOR_ENABLED=false as a string', function (): void { + config()->set('redactor.profiles.env_disabled', envShapedProfile(['enabled' => 'false'])); + + expect(RedactorConfig::fromConfig('env_disabled')->enabled)->toBeFalse(); + }); + + it('accepts the scan max file size as a string', function (): void { + // Config::integer() threw on this, so setting the documented + // REDACTOR_SCAN_MAX_FILE_SIZE made redactor:scan fail outright. + config()->set('redactor.scan.max_file_size', '1024'); + + $size = ConfigValue::positiveInt( + config('redactor.scan.max_file_size'), + 10_485_760, + 'scan.max_file_size' + ); + + expect($size)->toBe(1024); + + $dir = sys_get_temp_dir().'/redactor_env_'.uniqid(); + mkdir($dir); + file_put_contents($dir.'/small.txt', str_repeat('a', 10)); + file_put_contents($dir.'/big.txt', str_repeat('a', 2048)); + + $files = FileCollector::collect([$dir], [], $size); + + expect($files)->toHaveCount(1) + ->and(basename($files[0]))->toBe('small.txt'); + + cleanupDirectory($dir); + }); + + it('rejects an unknown non_redactable_object_behavior rather than ignoring it', function (): void { + config()->set('redactor.profiles.env_bad_behavior', envShapedProfile([ + 'non_redactable_object_behavior' => 'delete_everything', + ])); + + expect(fn (): RedactorConfig => RedactorConfig::fromConfig('env_bad_behavior')) + ->toThrow(\InvalidArgumentException::class, 'non_redactable_object_behavior'); + }); + + it('rejects a non-numeric entropy threshold', function (): void { + config()->set('redactor.profiles.env_bad_threshold', envShapedProfile([ + 'shannon_entropy' => ['enabled' => 'true', 'threshold' => 'high'], + ])); + + expect(fn (): RedactorConfig => RedactorConfig::fromConfig('env_bad_threshold')) + ->toThrow(\InvalidArgumentException::class, 'shannon_entropy.threshold'); + }); +}); + +describe('ConfigValue coercion', function (): void { + it('accepts the truthy and falsy spellings env files use', function (): void { + foreach (['true', 'TRUE', '1', 'yes', 'on', true, 1] as $truthy) { + expect(ConfigValue::bool($truthy, false, 'p'))->toBeTrue(); + } + + foreach (['false', 'FALSE', '0', 'no', 'off', '', false, 0] as $falsy) { + expect(ConfigValue::bool($falsy, true, 'p'))->toBeFalse(); + } + }); + + it('rejects strings that only look numeric', function (): void { + expect(fn (): int => ConfigValue::positiveInt('12abc', 1, 'p')) + ->toThrow(\InvalidArgumentException::class) + ->and(fn (): int => ConfigValue::positiveInt('1.5', 1, 'p')) + ->toThrow(\InvalidArgumentException::class); + }); + + it('falls back to the default when the value is absent', function (): void { + expect(ConfigValue::bool(null, true, 'p'))->toBeTrue() + ->and(ConfigValue::string(null, 'x', 'p'))->toBe('x') + ->and(ConfigValue::positiveInt(null, 7, 'p'))->toBe(7) + ->and(ConfigValue::float(null, 1.5, 'p'))->toBe(1.5) + ->and(ConfigValue::stringList(null, 'p'))->toBe([]); + }); + + it('drops non-string entries from string lists', function (): void { + expect(ConfigValue::stringList(['a', 1, null, 'b', []], 'p'))->toBe(['a', 'b']); + }); + + it('names the offending config path in every message', function (): void { + expect(fn (): bool => ConfigValue::bool('maybe', true, 'profiles.x.enabled')) + ->toThrow(\InvalidArgumentException::class, 'profiles.x.enabled'); + }); +}); + +describe('ConfigValue coercion of the other shapes', function (): void { + it('stringifies a number, since a replacement of 0 is still a replacement', function (): void { + expect(ConfigValue::string(5, 'x', 'p'))->toBe('5') + ->and(ConfigValue::string(1.5, 'x', 'p'))->toBe('1.5'); + }); + + it('rejects anything else as a string and says what it got', function (): void { + expect(fn (): string => ConfigValue::string(true, 'x', 'p')) + ->toThrow(ConfigurationException::class, 'got true') + ->and(fn (): string => ConfigValue::string(['a'], 'x', 'p')) + ->toThrow(ConfigurationException::class, 'got array') + ->and(fn (): string => ConfigValue::string(new \stdClass, 'x', 'p')) + ->toThrow(ConfigurationException::class, 'got stdClass'); + }); + + it('describes a scalar of the wrong kind with its type', function (): void { + expect(fn (): bool => ConfigValue::bool(2, true, 'p')) + ->toThrow(ConfigurationException::class, 'got integer(2)') + ->and(fn (): bool => ConfigValue::bool(1.5, true, 'p')) + ->toThrow(ConfigurationException::class, 'got double(1.5)') + ->and(fn (): array => ConfigValue::map('nope', 'p')) + ->toThrow(ConfigurationException::class, 'got string("nope")'); + }); + + it('reads a missing map as empty and a whole-number float as an integer', function (): void { + expect(ConfigValue::map(null, 'p'))->toBe([]) + ->and(ConfigValue::positiveInt(3.0, 1, 'p'))->toBe(3); + }); +}); diff --git a/tests/Feature/RedactorFacadeTest.php b/tests/Feature/RedactorFacadeTest.php index 2fb41f5..409cd34 100644 --- a/tests/Feature/RedactorFacadeTest.php +++ b/tests/Feature/RedactorFacadeTest.php @@ -5,16 +5,18 @@ namespace Tests\Feature; use Kirschbaum\Redactor\Facades\Redactor; +use Kirschbaum\Redactor\Strategies\BlockedKeysStrategy; +use Kirschbaum\Redactor\Strategies\SafeKeysStrategy; -describe('Redactor Facade Tests', function () { - beforeEach(function () { +describe('Redactor Facade Tests', function (): void { + beforeEach(function (): void { // Set up basic profile for facade testing config()->set('redactor.default_profile', 'facade_test'); config()->set('redactor.profiles.facade_test', [ 'enabled' => true, 'strategies' => [ - \Kirschbaum\Redactor\Strategies\SafeKeysStrategy::class, - \Kirschbaum\Redactor\Strategies\BlockedKeysStrategy::class, + SafeKeysStrategy::class, + BlockedKeysStrategy::class, ], 'safe_keys' => ['id'], 'blocked_keys' => ['password'], @@ -30,7 +32,7 @@ ]); }); - test('facade can redact data using default profile', function () { + test('facade can redact data using default profile', function (): void { $data = [ 'id' => 123, 'password' => 'secret123', @@ -43,12 +45,12 @@ ->and($result['_redacted'])->toBeTrue(); }); - test('facade can redact data using specific profile', function () { + test('facade can redact data using specific profile', function (): void { // Set up a different profile config()->set('redactor.profiles.strict_test', [ 'enabled' => true, 'strategies' => [ - \Kirschbaum\Redactor\Strategies\BlockedKeysStrategy::class, + BlockedKeysStrategy::class, ], 'safe_keys' => [], 'blocked_keys' => ['id', 'password'], @@ -75,19 +77,19 @@ ->and($result['_redacted'])->toBeTrue(); }); - test('facade can get available profiles', function () { - $profiles = Redactor::getAvailableProfiles(); + test('facade can get available profiles', function (): void { + $profiles = Redactor::profiles(); expect($profiles)->toBeArray() ->and($profiles)->toContain('facade_test'); }); - test('facade can check if profile exists', function () { - expect(Redactor::profileExists('facade_test'))->toBeTrue() - ->and(Redactor::profileExists('non_existent'))->toBeFalse(); + test('facade can check if profile exists', function (): void { + expect(Redactor::hasProfile('facade_test'))->toBeTrue() + ->and(Redactor::hasProfile('non_existent'))->toBeFalse(); }); - test('facade provides fresh instances to avoid state conflicts', function () { + test('facade provides fresh instances to avoid state conflicts', function (): void { // This test ensures that multiple facade calls don't interfere with each other $data1 = ['id' => 1, 'password' => 'secret1']; $data2 = ['id' => 2, 'password' => 'secret2']; diff --git a/tests/Feature/RedactorFailSafeTest.php b/tests/Feature/RedactorFailSafeTest.php new file mode 100644 index 0000000..79cd9c8 --- /dev/null +++ b/tests/Feature/RedactorFailSafeTest.php @@ -0,0 +1,208 @@ + resolve(Redactor::class)->redact(['a' => 1], 'does_not_exist')) + ->toThrow(ProfileNotFoundException::class, 'Redaction profile [does_not_exist] is not configured.'); + }); + + it('does not throw from redactSafely() for an unknown profile', function (): void { + $result = resolve(Redactor::class)->redactSafely(['secret' => 'value'], 'does_not_exist'); + + expect($result)->toBe('[REDACTED] (redaction failed)'); + }); + + it('replaces rather than passes through when redaction fails', function (): void { + // The whole point: a failure must not emit the payload it could not + // verify as safe. + $result = resolve(Redactor::class)->redactSafely( + ['password' => 'hunter2', 'card' => '4111111111111111'], + 'does_not_exist' + ); + + expect($result)->toBeString() + ->and($result)->not->toContain('hunter2') + ->and($result)->not->toContain('4111111111111111'); + }); + + it('survives a strategy that throws mid-redaction', function (): void { + config()->set('redactor.custom_strategies', ['exploding' => ExplodingStrategy::class]); + config()->set('redactor.profiles.exploding', [ + 'enabled' => true, + 'strategies' => ['exploding'], + 'safe_keys' => [], + 'blocked_keys' => [], + 'patterns' => [], + 'replacement' => '[REDACTED]', + 'mark_redacted' => false, + 'track_redacted_keys' => false, + 'non_redactable_object_behavior' => 'preserve', + 'max_value_length' => null, + 'redact_large_objects' => false, + 'max_object_size' => 100, + 'shannon_entropy' => ['enabled' => false], + ]); + + $result = resolve(Redactor::class)->redactSafely(['password' => 'hunter2'], 'exploding'); + + expect($result)->toBe('[REDACTED] (redaction failed)') + ->and($result)->not->toContain('hunter2'); + }); + + it('uses the profile replacement string in the failure marker when it can', function (): void { + config()->set('redactor.custom_strategies', ['exploding' => ExplodingStrategy::class]); + config()->set('redactor.profiles.exploding_masked', [ + 'enabled' => true, + 'strategies' => ['exploding'], + 'safe_keys' => [], + 'blocked_keys' => [], + 'patterns' => [], + 'replacement' => '***', + 'mark_redacted' => false, + 'track_redacted_keys' => false, + 'non_redactable_object_behavior' => 'preserve', + 'max_value_length' => null, + 'redact_large_objects' => false, + 'max_object_size' => 100, + 'shannon_entropy' => ['enabled' => false], + ]); + + expect(resolve(Redactor::class)->redactSafely(['password' => 'x'], 'exploding_masked')) + ->toBe('*** (redaction failed)'); + }); + + it('keeps the log channel alive when the configured profile is broken', function (): void { + config()->set('redactor.default_profile', 'missing_profile'); + + $formatter = new RedactorFormatter; + + // Previously this propagated InvalidArgumentException out of Monolog and + // killed every subsequent write to the channel. + $output = $formatter->format(record('user bob@example.com signed in', ['password' => 'hunter2'])); + + expect($output)->toBeString() + ->and($output)->toContain('testing.INFO') + ->and($output)->not->toContain('hunter2') + ->and($output)->not->toContain('bob@example.com'); + }); + + it('does not re-enter the logger while reporting its own failure', function (): void { + expect(InternalLog::isEmitting())->toBeFalse(); + + $seen = []; + + // A logger that calls back into redaction is exactly the re-entrancy + // that used to loop until the stack ran out. + Log::listen(function ($message) use (&$seen): void { + $seen[] = $message->message; + InternalLog::warning('nested diagnostic'); + }); + + resolve(Redactor::class)->redactSafely(['a' => 1], 'does_not_exist'); + + expect($seen)->toHaveCount(1) + ->and(InternalLog::isEmitting())->toBeFalse(); + }); + + it('swallows a logger that throws while reporting a failure', function (): void { + Log::listen(function (): void { + throw new \RuntimeException('logger is down'); + }); + + $result = resolve(Redactor::class)->redactSafely(['a' => 1], 'does_not_exist'); + + expect($result)->toBe('[REDACTED] (redaction failed)'); + }); +}); + +describe('redactor:validate', function (): void { + it('passes when every profile resolves', function (): void { + $this->artisan('redactor:validate') + ->assertSuccessful(); + }); + + it('fails and names a profile whose config is invalid', function (): void { + config()->set('redactor.profiles.broken', [ + 'enabled' => true, + 'strategies' => [BlockedKeysStrategy::class], + 'max_object_size' => 'lots', + 'shannon_entropy' => ['enabled' => false], + ]); + + $this->artisan('redactor:validate') + ->expectsOutputToContain('broken') + ->assertFailed(); + }); + + it('fails when a profile lists a strategy that cannot be resolved', function (): void { + config()->set('redactor.profiles.ghost', [ + 'enabled' => true, + 'strategies' => ['App\\Nope\\NotARealStrategy'], + 'safe_keys' => [], + 'blocked_keys' => [], + 'patterns' => [], + 'replacement' => '[REDACTED]', + 'mark_redacted' => false, + 'track_redacted_keys' => false, + 'non_redactable_object_behavior' => 'preserve', + 'max_value_length' => null, + 'redact_large_objects' => false, + 'max_object_size' => 100, + 'shannon_entropy' => ['enabled' => false], + ]); + + $this->artisan('redactor:validate') + ->expectsOutputToContain('ghost') + ->assertFailed(); + }); + + it('reports no profiles as a failure rather than a pass', function (): void { + config()->set('redactor.profiles', []); + + $this->artisan('redactor:validate')->assertFailed(); + }); +}); diff --git a/tests/Feature/RedactorFakeTest.php b/tests/Feature/RedactorFakeTest.php new file mode 100644 index 0000000..b0d46a6 --- /dev/null +++ b/tests/Feature/RedactorFakeTest.php @@ -0,0 +1,91 @@ +toBe($fake) + ->and(Redactor::redact(['password' => 'hunter2', 'id' => 1]))->toBe(['password' => '[REDACTED]', 'id' => 1, '_redacted' => true]); + + $fake->assertCalled(1); + $fake->assertRedacted('password'); + $fake->assertNotRedacted('id'); + $fake->assertFinding('blocked_key'); + $fake->assertSomethingRedacted(); + }); + + it('proves a secret never left, across every call and profile', function (): void { + $fake = Redactor::fake(); + + Redactor::redact('token sk_live_4eC39HqLyjWDarjtT1zdp7dc here'); + Redactor::redact(['note' => 'mail bob@example.com'], 'strict'); + Redactor::redactSafely(['nested' => ['password' => 'hunter2']]); + + $fake->assertNeverEmitted('sk_live_4eC39HqLyjWDarjtT1zdp7dc', 'bob@example.com', 'hunter2'); + $fake->assertProfileUsed('strict'); + $fake->assertCalled(3); + }); + + it('fails loudly when a secret did get out', function (): void { + $fake = Redactor::fake(); + + Redactor::redact(['comment' => 'my pin is 1234']); + + expect(fn () => $fake->assertNeverEmitted('1234'))->toThrow(AssertionFailedError::class, 'should have been redacted'); + }); + + it('fails when nothing was redacted but something should have been', function (): void { + $fake = Redactor::fake(); + + Redactor::redact(['plain' => 'text']); + + expect(fn () => $fake->assertRedacted('plain'))->toThrow(AssertionFailedError::class, 'No redaction recorded') + ->and(fn () => $fake->assertSomethingRedacted())->toThrow(AssertionFailedError::class); + + $fake->assertNothingRedacted(); + }); + + it('records what went through the Monolog processor', function (): void { + $fake = Redactor::fake(); + $processor = new RedactorProcessor(resolve(RedactorService::class)); + + $processor(new LogRecord(new DateTimeImmutable(true), 'app', Level::Info, 'user bob@example.com', ['password' => 'x'])); + + $fake->assertNeverEmitted('bob@example.com'); + $fake->assertRedacted('password'); + }); + + it('can forget and be asserted empty', function (): void { + $fake = Redactor::fake(); + Redactor::redact('x'); + $fake->forget(); + + $fake->assertNotCalled(); + expect($fake)->toBeInstanceOf(RedactorFake::class) + ->and($fake->recorded())->toBe([]); + }); +}); + +describe('Redactor::fake() findings', function (): void { + it('fails when no call produced a finding from the named rule', function (): void { + $fake = Redactor::fake(); + + Redactor::redact(['password' => 'hunter2']); + + expect(fn () => $fake->assertFinding('credit_card')) + ->toThrow(AssertionFailedError::class, 'No finding from rule [credit_card]'); + }); +}); diff --git a/tests/Feature/CustomLogTapTest.php b/tests/Feature/RedactorFormatterTapTest.php similarity index 70% rename from tests/Feature/CustomLogTapTest.php rename to tests/Feature/RedactorFormatterTapTest.php index 4dd1c1d..ced535a 100644 --- a/tests/Feature/CustomLogTapTest.php +++ b/tests/Feature/RedactorFormatterTapTest.php @@ -5,30 +5,30 @@ namespace Tests\Feature; use Illuminate\Log\Logger; -use Kirschbaum\Redactor\Logging\CustomLogTap; -use Kirschbaum\Redactor\Logging\ReadactFormatter; +use Kirschbaum\Redactor\Logging\RedactorFormatter; +use Kirschbaum\Redactor\Logging\RedactorFormatterTap; use Monolog\Handler\StreamHandler; use Monolog\Handler\TestHandler; use Monolog\Logger as MonologLogger; -describe('CustomLogTap Tests', function () { - test('tap applies ReadactFormatter to formattable handlers', function () { +describe('RedactorFormatterTap Tests', function (): void { + test('tap applies RedactorFormatter to formattable handlers', function (): void { // Create a logger with a formattable handler $monolog = new MonologLogger('test'); $handler = new TestHandler; $monolog->pushHandler($handler); $logger = new Logger($monolog); - $tap = new CustomLogTap; + $tap = new RedactorFormatterTap; // Apply the tap $tap($logger); // Verify the formatter was set - expect($handler->getFormatter())->toBeInstanceOf(ReadactFormatter::class); + expect($handler->getFormatter())->toBeInstanceOf(RedactorFormatter::class); }); - test('tap applies ReadactFormatter to multiple formattable handlers', function () { + test('tap applies RedactorFormatter to multiple formattable handlers', function (): void { // Create a logger with multiple formattable handlers $monolog = new MonologLogger('test'); $handler1 = new TestHandler; @@ -37,21 +37,21 @@ $monolog->pushHandler($handler2); $logger = new Logger($monolog); - $tap = new CustomLogTap; + $tap = new RedactorFormatterTap; // Apply the tap $tap($logger); // Verify formatters were set on both handlers - expect($handler1->getFormatter())->toBeInstanceOf(ReadactFormatter::class) - ->and($handler2->getFormatter())->toBeInstanceOf(ReadactFormatter::class); + expect($handler1->getFormatter())->toBeInstanceOf(RedactorFormatter::class) + ->and($handler2->getFormatter())->toBeInstanceOf(RedactorFormatter::class); }); - test('tap handles logger with no handlers gracefully', function () { + test('tap handles logger with no handlers gracefully', function (): void { // Create a logger with no handlers $monolog = new MonologLogger('test'); $logger = new Logger($monolog); - $tap = new CustomLogTap; + $tap = new RedactorFormatterTap; // This should not throw any exceptions $tap($logger); @@ -60,11 +60,11 @@ expect(true)->toBeTrue(); }); - test('tap skips non-formattable handlers', function () { + test('tap skips non-formattable handlers', function (): void { // Create a mock handler that doesn't implement FormattableHandlerInterface $nonFormattableHandler = new class { - public function getFormatter() + public function getFormatter(): null { return null; } @@ -77,7 +77,7 @@ public function getFormatter() // We need to use reflection to add the non-formattable handler // since Monolog validates handler types $logger = new Logger($monolog); - $tap = new CustomLogTap; + $tap = new RedactorFormatterTap; // Add only the formattable handler $monolog->pushHandler($formattableHandler); @@ -86,17 +86,17 @@ public function getFormatter() $tap($logger); // Only the formattable handler should have the formatter - expect($formattableHandler->getFormatter())->toBeInstanceOf(ReadactFormatter::class); + expect($formattableHandler->getFormatter())->toBeInstanceOf(RedactorFormatter::class); }); - test('tap can be invoked multiple times without issues', function () { + test('tap can be invoked multiple times without issues', function (): void { // Create a logger with a handler $monolog = new MonologLogger('test'); $handler = new TestHandler; $monolog->pushHandler($handler); $logger = new Logger($monolog); - $tap = new CustomLogTap; + $tap = new RedactorFormatterTap; // Apply the tap multiple times $tap($logger); @@ -104,6 +104,6 @@ public function getFormatter() $tap($logger); // Should still work and have the correct formatter - expect($handler->getFormatter())->toBeInstanceOf(ReadactFormatter::class); + expect($handler->getFormatter())->toBeInstanceOf(RedactorFormatter::class); }); }); diff --git a/tests/Feature/ReadactFormatterTest.php b/tests/Feature/RedactorFormatterTest.php similarity index 71% rename from tests/Feature/ReadactFormatterTest.php rename to tests/Feature/RedactorFormatterTest.php index 64dbb23..1c9e172 100644 --- a/tests/Feature/ReadactFormatterTest.php +++ b/tests/Feature/RedactorFormatterTest.php @@ -5,19 +5,22 @@ namespace Tests\Feature; use DateTimeImmutable; -use Kirschbaum\Redactor\Logging\ReadactFormatter; +use Kirschbaum\Redactor\Logging\RedactorFormatter; +use Kirschbaum\Redactor\Strategies\BlockedKeysStrategy; +use Kirschbaum\Redactor\Strategies\RegexPatternsStrategy; +use Monolog\Formatter\LineFormatter; use Monolog\Level; use Monolog\LogRecord; -describe('ReadactFormatter Tests', function () { - beforeEach(function () { +describe('RedactorFormatter Tests', function (): void { + beforeEach(function (): void { // Set up basic redaction profile for testing config()->set('redactor.default_profile', 'logging_test'); config()->set('redactor.profiles.logging_test', [ 'enabled' => true, 'strategies' => [ - \Kirschbaum\Redactor\Strategies\BlockedKeysStrategy::class, - \Kirschbaum\Redactor\Strategies\RegexPatternsStrategy::class, + BlockedKeysStrategy::class, + RegexPatternsStrategy::class, ], 'safe_keys' => ['id'], 'blocked_keys' => ['password', 'token', 'secret'], @@ -38,8 +41,8 @@ ]); }); - test('formats basic log record with string message', function () { - $formatter = new ReadactFormatter; + test('formats basic log record with string message', function (): void { + $formatter = new RedactorFormatter; $datetime = new DateTimeImmutable('2023-12-25 14:30:45.123456'); $record = new LogRecord( @@ -55,8 +58,8 @@ expect($result)->toBe("[2023-12-25 14:30:45.123456] app.INFO: User logged in successfully\n"); }); - test('redacts sensitive data in log message', function () { - $formatter = new ReadactFormatter; + test('redacts sensitive data in log message', function (): void { + $formatter = new RedactorFormatter; $datetime = new DateTimeImmutable('2023-12-25 14:30:45.123456'); $record = new LogRecord( @@ -74,8 +77,8 @@ ->and($result)->toContain('[2023-12-25 14:30:45.123456] app.ERROR:'); }); - test('handles array message by converting to json', function () { - $formatter = new ReadactFormatter; + test('handles array message by converting to json', function (): void { + $formatter = new RedactorFormatter; $datetime = new DateTimeImmutable('2023-12-25 14:30:45.123456'); $arrayMessage = ['action' => 'login', 'password' => 'secret123']; @@ -96,8 +99,8 @@ ->and($result)->not->toContain('secret123'); }); - test('handles object message by converting to json', function () { - $formatter = new ReadactFormatter; + test('handles object message by converting to json', function (): void { + $formatter = new RedactorFormatter; $datetime = new DateTimeImmutable('2023-12-25 14:30:45.123456'); $objectMessage = (object) ['action' => 'login', 'token' => 'abc123']; @@ -118,8 +121,8 @@ ->and($result)->not->toContain('abc123'); }); - test('formats log record with context data', function () { - $formatter = new ReadactFormatter; + test('formats log record with context data', function (): void { + $formatter = new RedactorFormatter; $datetime = new DateTimeImmutable('2023-12-25 14:30:45.123456'); $context = [ @@ -147,8 +150,8 @@ ->and($result)->toEndWith("\n"); }); - test('handles empty context gracefully', function () { - $formatter = new ReadactFormatter; + test('handles empty context gracefully', function (): void { + $formatter = new RedactorFormatter; $datetime = new DateTimeImmutable('2023-12-25 14:30:45.123456'); $record = new LogRecord( @@ -165,8 +168,8 @@ ->and($result)->not->toContain('{}'); }); - test('handles different log levels correctly', function () { - $formatter = new ReadactFormatter; + test('handles different log levels correctly', function (): void { + $formatter = new RedactorFormatter; $datetime = new DateTimeImmutable('2023-12-25 14:30:45.123456'); $levels = [ @@ -205,8 +208,8 @@ } }); - test('formatBatch returns formatted first record', function () { - $formatter = new ReadactFormatter; + test('formatBatch formats every record, not just the first', function (): void { + $formatter = new RedactorFormatter; $datetime = new DateTimeImmutable('2023-12-25 14:30:45.123456'); $records = [ @@ -228,12 +231,16 @@ $result = $formatter->formatBatch($records); - expect($result)->toBe("[2023-12-25 14:30:45.123456] app.INFO: First message\n") - ->and($result)->not->toContain('Second message'); + // Previously this returned format($records[0]), so every record but + // the first was silently dropped by any batching handler. + expect($result)->toBe( + "[2023-12-25 14:30:45.123456] app.INFO: First message\n" + ."[2023-12-25 14:30:45.123456] app.ERROR: Second message\n" + ); }); - test('handles null context values', function () { - $formatter = new ReadactFormatter; + test('handles null context values', function (): void { + $formatter = new RedactorFormatter; $datetime = new DateTimeImmutable('2023-12-25 14:30:45.123456'); $context = [ @@ -254,8 +261,8 @@ expect($result)->toContain('{"user_id":null,"action":"test"}'); }); - test('preserves microseconds in timestamp', function () { - $formatter = new ReadactFormatter; + test('preserves microseconds in timestamp', function (): void { + $formatter = new RedactorFormatter; $datetime = new DateTimeImmutable('2023-12-25 14:30:45.999999'); $record = new LogRecord( @@ -271,3 +278,30 @@ expect($result)->toContain('[2023-12-25 14:30:45.999999]'); }); }); + +describe('RedactorFormatter batches', function (): void { + test('formats a batch through the inner formatter with every record redacted', function (): void { + config()->set('redactor.default_profile', 'logging_test'); + config()->set('redactor.profiles.logging_test', [ + 'enabled' => true, + 'strategies' => [RegexPatternsStrategy::class], + 'safe_keys' => [], + 'blocked_keys' => [], + 'patterns' => ['password_pattern' => '/password:\s*\S+/i', 'token_pattern' => '/token:\s*\S+/i'], + 'replacement' => '[REDACTED]', + 'mark_redacted' => false, + 'track_redacted_keys' => false, + 'non_redactable_object_behavior' => 'preserve', + 'max_value_length' => null, + 'redact_large_objects' => false, + 'max_object_size' => 100, + 'shannon_entropy' => ['enabled' => false], + ]); + + $formatter = new RedactorFormatter(new LineFormatter("%message%\n")); + $record = fn (string $message): LogRecord => new LogRecord(new DateTimeImmutable, 'app', Level::Info, $message); + + expect($formatter->formatBatch([$record('login password: hunter2'), $record('sent token: abc')])) + ->toBe("login [REDACTED]\nsent [REDACTED]\n"); + }); +}); diff --git a/tests/Feature/RedactorInputTypesTest.php b/tests/Feature/RedactorInputTypesTest.php index db01bd8..3ce307d 100644 --- a/tests/Feature/RedactorInputTypesTest.php +++ b/tests/Feature/RedactorInputTypesTest.php @@ -5,20 +5,26 @@ namespace Tests\Feature; use Kirschbaum\Redactor\Redactor; - -describe('Redactor Mixed Input Types Tests', function () { - beforeEach(function () { +use Kirschbaum\Redactor\Strategies\BlockedKeysStrategy; +use Kirschbaum\Redactor\Strategies\LargeObjectStrategy; +use Kirschbaum\Redactor\Strategies\LargeStringStrategy; +use Kirschbaum\Redactor\Strategies\RegexPatternsStrategy; +use Kirschbaum\Redactor\Strategies\SafeKeysStrategy; +use Kirschbaum\Redactor\Strategies\ShannonEntropyStrategy; + +describe('Redactor Mixed Input Types Tests', function (): void { + beforeEach(function (): void { // Set up profile-based configuration config()->set('redactor.default_profile', 'default'); config()->set('redactor.profiles.default', [ 'enabled' => true, 'strategies' => [ - \Kirschbaum\Redactor\Strategies\SafeKeysStrategy::class, - \Kirschbaum\Redactor\Strategies\BlockedKeysStrategy::class, - \Kirschbaum\Redactor\Strategies\LargeObjectStrategy::class, - \Kirschbaum\Redactor\Strategies\LargeStringStrategy::class, - \Kirschbaum\Redactor\Strategies\RegexPatternsStrategy::class, - \Kirschbaum\Redactor\Strategies\ShannonEntropyStrategy::class, + SafeKeysStrategy::class, + BlockedKeysStrategy::class, + LargeObjectStrategy::class, + LargeStringStrategy::class, + RegexPatternsStrategy::class, + ShannonEntropyStrategy::class, ], 'safe_keys' => ['id', 'user_id'], 'blocked_keys' => ['password', 'secret'], @@ -41,7 +47,7 @@ ]); }); - it('handles string input with email pattern redaction', function () { + it('handles string input with email pattern redaction', function (): void { $redactor = new Redactor; $email = 'user@example.com'; @@ -50,7 +56,7 @@ expect($result)->toBe('[REDACTED]'); }); - it('handles string input without redaction needed', function () { + it('handles string input without redaction needed', function (): void { $redactor = new Redactor; $normalString = 'Hello World'; @@ -59,7 +65,7 @@ expect($result)->toBe('Hello World'); }); - it('handles string input with high entropy redaction', function () { + it('handles string input with high entropy redaction', function (): void { $redactor = new Redactor; // High entropy string over minimum length @@ -69,7 +75,7 @@ expect($result)->toBe('[REDACTED]'); }); - it('handles object input with toArray method', function () { + it('handles object input with toArray method', function (): void { $redactor = new Redactor; $object = new class @@ -93,7 +99,7 @@ public function toArray(): array ->and($result['_redacted'])->toBeTrue(); }); - it('handles object input without toArray method via JSON serialization', function () { + it('handles object input without toArray method via JSON serialization', function (): void { $redactor = new Redactor; $object = new \stdClass; @@ -110,7 +116,7 @@ public function toArray(): array ->and($result['_redacted'])->toBeTrue(); }); - it('handles non-serializable object based on behavior config', function () { + it('handles non-serializable object based on behavior config', function (): void { // Update the profile config for this test config()->set('redactor.profiles.default.non_redactable_object_behavior', 'preserve'); @@ -125,7 +131,7 @@ public function toArray(): array expect($result)->toBe($object); // Should preserve original object }); - it('handles non-serializable object with remove behavior', function () { + it('handles non-serializable object with remove behavior', function (): void { // Update the profile config for this test config()->set('redactor.profiles.default.non_redactable_object_behavior', 'remove'); @@ -140,7 +146,7 @@ public function toArray(): array expect($result)->toBe('__REDACTOR_REMOVE_OBJECT__'); }); - it('handles non-serializable object with empty_array behavior', function () { + it('handles non-serializable object with empty_array behavior', function (): void { // Update the profile config for this test config()->set('redactor.profiles.default.non_redactable_object_behavior', 'empty_array'); @@ -158,7 +164,7 @@ public function toArray(): array ->and($result['_redacted'])->toBeTrue(); }); - it('handles non-serializable object with redact behavior', function () { + it('handles non-serializable object with redact behavior', function (): void { // Update the profile config for this test config()->set('redactor.profiles.default.non_redactable_object_behavior', 'redact'); @@ -175,7 +181,7 @@ public function toArray(): array ->and($result)->toContain('stdClass'); }); - it('handles integer input unchanged', function () { + it('handles integer input unchanged', function (): void { $redactor = new Redactor; $integer = 12345; @@ -184,7 +190,7 @@ public function toArray(): array expect($result)->toBe(12345); }); - it('handles float input unchanged', function () { + it('handles float input unchanged', function (): void { $redactor = new Redactor; $float = 123.45; @@ -193,7 +199,7 @@ public function toArray(): array expect($result)->toBe(123.45); }); - it('handles boolean input unchanged', function () { + it('handles boolean input unchanged', function (): void { $redactor = new Redactor; $boolean = true; @@ -202,7 +208,7 @@ public function toArray(): array expect($result)->toBe(true); }); - it('handles null input unchanged', function () { + it('handles null input unchanged', function (): void { $redactor = new Redactor; $null = null; @@ -211,7 +217,7 @@ public function toArray(): array expect($result)->toBeNull(); }); - it('handles array input with metadata (existing functionality)', function () { + it('handles array input with metadata (existing functionality)', function (): void { // Update the profile config for this test config()->set('redactor.profiles.default.track_redacted_keys', true); @@ -234,7 +240,7 @@ public function toArray(): array ->and($result['_redacted_keys'])->toContain('password'); }); - it('does not add metadata to non-array results', function () { + it('does not add metadata to non-array results', function (): void { $redactor = new Redactor; $email = 'user@example.com'; @@ -245,7 +251,7 @@ public function toArray(): array ->and($result)->not->toBeArray(); }); - it('handles nested mixed types within arrays', function () { + it('handles nested mixed types within arrays', function (): void { $redactor = new Redactor; $object = new \stdClass; @@ -277,19 +283,19 @@ public function toArray(): array }); }); -describe('Redactor Nested Structure Tests', function () { - beforeEach(function () { +describe('Redactor Nested Structure Tests', function (): void { + beforeEach(function (): void { // Set up profile-based configuration for nested tests config()->set('redactor.default_profile', 'default'); config()->set('redactor.profiles.default', [ 'enabled' => true, 'strategies' => [ - \Kirschbaum\Redactor\Strategies\SafeKeysStrategy::class, - \Kirschbaum\Redactor\Strategies\BlockedKeysStrategy::class, - \Kirschbaum\Redactor\Strategies\LargeObjectStrategy::class, - \Kirschbaum\Redactor\Strategies\LargeStringStrategy::class, - \Kirschbaum\Redactor\Strategies\RegexPatternsStrategy::class, - \Kirschbaum\Redactor\Strategies\ShannonEntropyStrategy::class, + SafeKeysStrategy::class, + BlockedKeysStrategy::class, + LargeObjectStrategy::class, + LargeStringStrategy::class, + RegexPatternsStrategy::class, + ShannonEntropyStrategy::class, ], 'safe_keys' => ['id', 'name'], 'blocked_keys' => ['password', 'secret'], @@ -310,7 +316,7 @@ public function toArray(): array ]); }); - it('handles nested arrays and objects recursively', function () { + it('handles nested arrays and objects recursively', function (): void { $redactor = new Redactor; $context = [ diff --git a/tests/Feature/RedactorIntegrationTest.php b/tests/Feature/RedactorIntegrationTest.php index 263fc2e..cac9d17 100644 --- a/tests/Feature/RedactorIntegrationTest.php +++ b/tests/Feature/RedactorIntegrationTest.php @@ -5,17 +5,21 @@ namespace Tests\Feature; use Kirschbaum\Redactor\Redactor; +use Kirschbaum\Redactor\Strategies\BlockedKeysStrategy; +use Kirschbaum\Redactor\Strategies\RegexPatternsStrategy; +use Kirschbaum\Redactor\Strategies\SafeKeysStrategy; +use Kirschbaum\Redactor\Strategies\ShannonEntropyStrategy; -describe('Redactor Integration Tests', function () { - it('integrates with Laravel Log and redacts context', function () { +describe('Redactor Integration Tests', function (): void { + it('integrates with Laravel Log and redacts context', function (): void { // Set up completely explicit profile for this specific test config()->set('redactor.default_profile', 'integration_test'); config()->set('redactor.profiles.integration_test', [ 'enabled' => true, 'strategies' => [ - \Kirschbaum\Redactor\Strategies\SafeKeysStrategy::class, - \Kirschbaum\Redactor\Strategies\BlockedKeysStrategy::class, - \Kirschbaum\Redactor\Strategies\RegexPatternsStrategy::class, + SafeKeysStrategy::class, + BlockedKeysStrategy::class, + RegexPatternsStrategy::class, ], 'safe_keys' => ['user_id'], 'blocked_keys' => ['password', 'secret'], @@ -59,16 +63,16 @@ }); }); -describe('Redactor Real-world Scenario Tests', function () { - it('handles realistic user registration context', function () { +describe('Redactor Real-world Scenario Tests', function (): void { + it('handles realistic user registration context', function (): void { // Explicit profile for user registration test config()->set('redactor.default_profile', 'user_registration_test'); config()->set('redactor.profiles.user_registration_test', [ 'enabled' => true, 'strategies' => [ - \Kirschbaum\Redactor\Strategies\SafeKeysStrategy::class, - \Kirschbaum\Redactor\Strategies\BlockedKeysStrategy::class, - \Kirschbaum\Redactor\Strategies\RegexPatternsStrategy::class, + SafeKeysStrategy::class, + BlockedKeysStrategy::class, + RegexPatternsStrategy::class, ], 'safe_keys' => ['id', 'user_id', 'created_at', 'updated_at'], 'blocked_keys' => ['email', 'ssn', 'password'], @@ -116,15 +120,15 @@ ->and($result['_redacted'])->toBeTrue(); }); - it('handles API request context with tokens', function () { + it('handles API request context with tokens', function (): void { // Explicit profile for API token test with Shannon entropy enabled config()->set('redactor.default_profile', 'api_token_test'); config()->set('redactor.profiles.api_token_test', [ 'enabled' => true, 'strategies' => [ - \Kirschbaum\Redactor\Strategies\SafeKeysStrategy::class, - \Kirschbaum\Redactor\Strategies\BlockedKeysStrategy::class, - \Kirschbaum\Redactor\Strategies\ShannonEntropyStrategy::class, + SafeKeysStrategy::class, + BlockedKeysStrategy::class, + ShannonEntropyStrategy::class, ], 'safe_keys' => ['request_id', 'user_id', 'endpoint', 'method', 'created_at'], 'blocked_keys' => ['api_key'], @@ -170,15 +174,15 @@ ->and($result['_redacted'])->toBeTrue(); }); - it('handles complex e-commerce order context', function () { + it('handles complex e-commerce order context', function (): void { // Explicit profile for e-commerce test config()->set('redactor.default_profile', 'ecommerce_test'); config()->set('redactor.profiles.ecommerce_test', [ 'enabled' => true, 'strategies' => [ - \Kirschbaum\Redactor\Strategies\SafeKeysStrategy::class, - \Kirschbaum\Redactor\Strategies\BlockedKeysStrategy::class, - \Kirschbaum\Redactor\Strategies\ShannonEntropyStrategy::class, + SafeKeysStrategy::class, + BlockedKeysStrategy::class, + ShannonEntropyStrategy::class, ], 'safe_keys' => ['order_id', 'user_id', 'created_at', 'address', 'city', 'name', 'price', 'amount', 'payment_id'], 'blocked_keys' => ['email', 'ssn'], @@ -238,15 +242,15 @@ ->and($result['_redacted'])->toBeTrue(); }); - it('handles logging context with database queries and errors', function () { + it('handles logging context with database queries and errors', function (): void { // Explicit profile for logging test config()->set('redactor.default_profile', 'logging_test'); config()->set('redactor.profiles.logging_test', [ 'enabled' => true, 'strategies' => [ - \Kirschbaum\Redactor\Strategies\SafeKeysStrategy::class, - \Kirschbaum\Redactor\Strategies\BlockedKeysStrategy::class, - \Kirschbaum\Redactor\Strategies\ShannonEntropyStrategy::class, + SafeKeysStrategy::class, + BlockedKeysStrategy::class, + ShannonEntropyStrategy::class, ], 'safe_keys' => ['user_id', 'query', 'level', 'message', 'file', 'line', 'duration_ms'], 'blocked_keys' => ['password', 'api_key'], diff --git a/tests/Feature/RedactorKeyMatcherTest.php b/tests/Feature/RedactorKeyMatcherTest.php new file mode 100644 index 0000000..7cd53ea --- /dev/null +++ b/tests/Feature/RedactorKeyMatcherTest.php @@ -0,0 +1,211 @@ + true, + 'strategies' => [BlockedKeysStrategy::class], + 'safe_keys' => [], + 'blocked_keys' => $blockedKeys, + 'patterns' => [], + 'replacement' => '[REDACTED]', + 'mark_redacted' => false, + 'track_redacted_keys' => false, + 'non_redactable_object_behavior' => 'preserve', + 'max_value_length' => null, + 'redact_large_objects' => false, + 'max_object_size' => 100, + 'shannon_entropy' => ['enabled' => false], + ]; +} + +describe('KeyMatcher pattern shapes', function (): void { + afterEach(fn () => KeyMatcher::flush()); + + it('matches exact names case-insensitively', function (): void { + $matcher = KeyMatcher::for(['password']); + + expect($matcher->matches('password'))->toBeTrue() + ->and($matcher->matches('PASSWORD'))->toBeTrue() + ->and($matcher->matches('Password'))->toBeTrue() + ->and($matcher->matches('password_hint'))->toBeFalse(); + }); + + it('matches contains patterns', function (): void { + $matcher = KeyMatcher::for(['*token*']); + + expect($matcher->matches('token'))->toBeTrue() + ->and($matcher->matches('api_token'))->toBeTrue() + ->and($matcher->matches('token_data'))->toBeTrue() + ->and($matcher->matches('my_TOKEN_field'))->toBeTrue() + ->and($matcher->matches('tokn'))->toBeFalse(); + }); + + it('matches prefix patterns', function (): void { + $matcher = KeyMatcher::for(['password*']); + + expect($matcher->matches('password'))->toBeTrue() + ->and($matcher->matches('password_confirmation'))->toBeTrue() + ->and($matcher->matches('user_password'))->toBeFalse(); + }); + + it('matches suffix patterns', function (): void { + $matcher = KeyMatcher::for(['*_key']); + + expect($matcher->matches('private_key'))->toBeTrue() + ->and($matcher->matches('signing_key'))->toBeTrue() + ->and($matcher->matches('key_id'))->toBeFalse(); + }); + + it('matches multi-wildcard patterns via the regex path', function (): void { + $matcher = KeyMatcher::for(['user_*_token']); + + expect($matcher->matches('user_api_token'))->toBeTrue() + ->and($matcher->matches('user_auth_token'))->toBeTrue() + ->and($matcher->matches('user_token'))->toBeFalse() + ->and($matcher->matches('admin_api_token'))->toBeFalse(); + }); + + it('treats a lone asterisk as matching everything', function (): void { + $matcher = KeyMatcher::for(['*']); + + expect($matcher->matches('anything'))->toBeTrue() + ->and($matcher->matches('x'))->toBeTrue(); + }); + + it('never matches the empty key', function (): void { + expect(KeyMatcher::for(['*'])->matches(''))->toBeFalse() + ->and(KeyMatcher::for([''])->matches(''))->toBeFalse(); + }); + + it('reports an empty pattern list as empty and matches nothing', function (): void { + $matcher = KeyMatcher::for([]); + + expect($matcher->isEmpty())->toBeTrue() + ->and($matcher->matches('password'))->toBeFalse(); + }); + + it('combines exact and wildcard patterns in one list', function (): void { + $matcher = KeyMatcher::for(['password', '*token*', 'user_*_data']); + + expect($matcher->matches('password'))->toBeTrue() + ->and($matcher->matches('api_token'))->toBeTrue() + ->and($matcher->matches('user_profile_data'))->toBeTrue() + ->and($matcher->matches('normal_field'))->toBeFalse(); + }); + + it('reuses the compiled matcher for an identical pattern list', function (): void { + expect(KeyMatcher::for(['a', '*b*']))->toBe(KeyMatcher::for(['a', '*b*'])) + ->and(KeyMatcher::for(['a', '*b*']))->not->toBe(KeyMatcher::for(['a', '*c*'])); + }); +}); + +describe('Blocked keys behaviour is unchanged by compilation', function (): void { + afterEach(fn () => KeyMatcher::flush()); + + it('matches the same keys through the full redactor', function (): void { + config()->set('redactor.profiles.blocked', blockedProfile([ + 'password', + '*token*', + '*key*', + 'user_*_data', + ])); + + $result = resolve(Redactor::class)->redact([ + 'user_id' => 123, + 'api_token' => 'secret123', + 'access_token' => 'abc123', + 'my_custom_token' => 'xyz789', + 'user_api_key' => 'key123', + 'private_key_data' => 'private', + 'password' => 'secret', + 'user_profile_data' => 'profile', + 'user_settings_data' => 'settings', + 'normal_field' => 'safe_value', + ], 'blocked'); + + expect($result)->toBe([ + 'user_id' => 123, + 'api_token' => '[REDACTED]', + 'access_token' => '[REDACTED]', + 'my_custom_token' => '[REDACTED]', + 'user_api_key' => '[REDACTED]', + 'private_key_data' => '[REDACTED]', + 'password' => '[REDACTED]', + 'user_profile_data' => '[REDACTED]', + 'user_settings_data' => '[REDACTED]', + 'normal_field' => 'safe_value', + ]); + }); + + it('picks up a changed blocked_keys list rather than serving a stale matcher', function (): void { + config()->set('redactor.profiles.blocked', blockedProfile(['password'])); + + expect(resolve(Redactor::class)->redact(['secret' => 'v'], 'blocked')) + ->toBe(['secret' => 'v']); + + config()->set('redactor.profiles.blocked', blockedProfile(['password', 'secret'])); + + expect(resolve(Redactor::class)->redact(['secret' => 'v'], 'blocked')) + ->toBe(['secret' => '[REDACTED]']); + }); +}); + +describe('Compiled key matchers live on the profile', function (): void { + afterEach(fn () => KeyMatcher::flush()); + + it('resolves both matchers once with the profile', function (): void { + config()->set('redactor.profiles.held', blockedProfile(['password', '*token*'])); + + $config = RedactorConfig::fromConfig('held'); + + expect($config->safeKeyMatcher)->toBeInstanceOf(KeyMatcher::class) + ->and($config->blockedKeyMatcher)->toBeInstanceOf(KeyMatcher::class) + ->and($config->blockedKeyMatcher->matches('api_token'))->toBeTrue() + ->and($config->blockedKeyMatcher->matches('harmless'))->toBeFalse(); + }); + + it('still reflects a changed key list', function (): void { + // The matcher is resolved with the profile, so a config change has to + // produce a new profile and a new matcher - otherwise a security + // setting would silently stop taking effect. + config()->set('redactor.profiles.held', blockedProfile(['password'])); + + expect(resolve(Redactor::class)->redact(['secret' => 'v'], 'held'))->toBe(['secret' => 'v']); + + config()->set('redactor.profiles.held', blockedProfile(['password', 'secret'])); + + expect(resolve(Redactor::class)->redact(['secret' => 'v'], 'held'))->toBe(['secret' => '[REDACTED]']); + }); + + it('builds matchers for a directly constructed config too', function (): void { + $config = new RedactorConfig( + enabled: true, + safeKeys: ['keep'], + blockedKeys: ['drop'], + patterns: [], + replacement: '[REDACTED]', + markRedacted: false, + trackRedactedKeys: false, + nonRedactableObjectBehavior: 'preserve', + maxValueLength: null, + redactLargeObjects: false, + maxObjectSize: 100, + shannonEntropy: ['enabled' => false], + strategies: [], + profile: 'manual', + ); + + expect($config->safeKeyMatcher->matches('keep'))->toBeTrue() + ->and($config->blockedKeyMatcher->matches('drop'))->toBeTrue(); + }); +}); diff --git a/tests/Feature/RedactorKnownSecretsTest.php b/tests/Feature/RedactorKnownSecretsTest.php new file mode 100644 index 0000000..a99c993 --- /dev/null +++ b/tests/Feature/RedactorKnownSecretsTest.php @@ -0,0 +1,104 @@ + true, + 'strategies' => [KnownSecretsStrategy::class, RegexPatternsStrategy::class], + 'safe_keys' => [], + 'blocked_keys' => [], + 'patterns' => [], + 'known_secrets' => ['values' => ['s3cr3t-value-1'], 'config' => []], + 'operators' => ['default' => 'redact'], + 'replacement' => '[REDACTED]', + 'mark_redacted' => false, + 'track_redacted_keys' => false, + 'non_redactable_object_behavior' => 'preserve', + 'max_value_length' => null, + 'redact_large_objects' => false, + 'max_object_size' => 100, + 'pseudonymization' => ['key' => testPseudonymizationKey()], + 'shannon_entropy' => ['enabled' => false], + ], $overrides); +} + +describe('Known secrets', function (): void { + beforeEach(fn () => config()->set('redactor.profiles.known', knownSecretsProfile())); + + it('redacts a configured value wherever it appears verbatim', function (): void { + $result = resolve(Redactor::class)->redact([ + 'msg' => 'called with s3cr3t-value-1 twice: s3cr3t-value-1', + 'json' => '{"token":"s3cr3t-value-1"}', + ], 'known'); + + expect($result['msg'])->toBe('called with [REDACTED] twice: [REDACTED]') + ->and($result['json'])->toBe('{"token":"[REDACTED]"}'); + }); + + it('is case-sensitive, because secrets are', function (): void { + expect(resolve(Redactor::class)->redact('S3CR3T-VALUE-1', 'known'))->toBe('S3CR3T-VALUE-1'); + }); + + it('reads secrets from config keys, including every string under an array', function (): void { + config()->set('services.acme', ['key' => 'acme-key-12345', 'secret' => 'acme-secret-67890', 'enabled' => true, 'retries' => 3]); + config()->set('redactor.profiles.known.known_secrets', ['config' => ['services.acme', 'app.missing']]); + + expect(resolve(Redactor::class)->redact('acme-key-12345 / acme-secret-67890', 'known')) + ->toBe('[REDACTED] / [REDACTED]'); + }); + + it('refuses values too short to match safely', function (): void { + $registry = new SecretRegistry; + + expect($registry->add('short'))->toBeFalse() + ->and($registry->add('long-enough'))->toBeTrue() + ->and($registry->count())->toBe(1); + }); + + it('accepts secrets registered at runtime, for every profile', function (): void { + $redactor = resolve(Redactor::class); + $redactor->registerSecret('minted-at-runtime-token'); + + expect($redactor->redact('using minted-at-runtime-token now', 'known')) + ->toBe('using [REDACTED] now'); + }); + + it('goes through the operator for its entity', function (): void { + config()->set('redactor.profiles.known.operators', ['default' => 'redact', 'known_secret' => 'hash']); + + expect(resolve(Redactor::class)->redact('x s3cr3t-value-1 y', 'known')) + ->toMatch('/^x \[known_secret:[a-z0-9]+\] y$/'); + }); + + it('reports the finding as certain', function (): void { + $result = resolve(Redactor::class)->inspect('s3cr3t-value-1', 'known'); + + expect($result->findings[0]->rule)->toBe('known_secret') + ->and($result->findings[0]->confidence?->score)->toBe(1.0); + }); + + it('redacts APP_KEY in the shipped default profile', function (): void { + $key = 'base64:'.base64_encode(random_bytes(32)); + config()->set('app.key', $key); + + $result = resolve(Redactor::class)->redact(['note' => "leaked {$key} here"]); + + expect($result['note'])->not->toContain($key) + ->and($result['note'])->toStartWith('leaked '); + }); + + it('does not fail the profile when a configured secret is null', function (): void { + config()->set('redactor.profiles.known.known_secrets', ['config' => ['services.nothing.key']]); + + expect(resolve(Redactor::class)->redact('fine', 'known'))->toBe('fine'); + }); +}); diff --git a/tests/Feature/RedactorLargeStringTest.php b/tests/Feature/RedactorLargeStringTest.php new file mode 100644 index 0000000..20ca623 --- /dev/null +++ b/tests/Feature/RedactorLargeStringTest.php @@ -0,0 +1,87 @@ + true, + 'strategies' => [LargeStringStrategy::class, RegexPatternsStrategy::class], + 'safe_keys' => [], + 'blocked_keys' => [], + 'patterns' => ['email' => '/[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+/'], + 'replacement' => '[REDACTED]', + 'mark_redacted' => false, + 'track_redacted_keys' => false, + 'non_redactable_object_behavior' => 'preserve', + 'max_value_length' => 40, + 'redact_large_objects' => false, + 'max_object_size' => null, + 'shannon_entropy' => ['enabled' => false], + ], $overrides); +} + +describe('Long strings', function (): void { + beforeEach(function (): void { + config()->set('redactor.profiles.long', largeStringProfile()); + }); + + it('keeps the head of a long string and notes what was cut', function (): void { + $value = str_repeat('trace line. ', 10); // 120 bytes + + $result = resolve(Redactor::class)->inspect($value, 'long'); + + expect($result->value)->toStartWith(substr($value, 0, 40)) + ->and($result->value)->toEndWith('[REDACTED] (String truncated: 120 characters, 40 kept)') + ->and($result->wasRedacted)->toBeTrue() + ->and($result->findings[0]->rule)->toBe('large_string') + ->and($result->findings[0]->offset)->toBe(40) + ->and($result->findings[0]->length)->toBe(80); + }); + + it('still scans the head it keeps', function (): void { + $value = 'contact bob@example.com about '.str_repeat('x', 100); + + $result = resolve(Redactor::class)->redact($value, 'long'); + + expect($result)->toStartWith('contact [REDACTED] about ') + ->and($result)->not->toContain('bob@example.com'); + }); + + it('never splits a multibyte character at the cut', function (): void { + $value = str_repeat('é', 30); // 60 bytes, limit is 40 + + $result = resolve(Redactor::class)->redact($value, 'long'); + + $head = explode(' [REDACTED]', $result)[0]; + + expect(mb_check_encoding($head, 'UTF-8'))->toBeTrue() + ->and($head)->toBe(str_repeat('é', 20)); + }); + + it('replaces the whole value when the behaviour is redact', function (): void { + config()->set('redactor.profiles.long.large_string_behavior', 'redact'); + + $result = resolve(Redactor::class)->redact(str_repeat('a', 100), 'long'); + + expect($result)->toBe('[REDACTED] (String with 100 characters)'); + }); + + it('rejects an unknown behaviour', function (): void { + config()->set('redactor.profiles.long.large_string_behavior', 'shrug'); + + resolve(Redactor::class)->redact('x', 'long'); + })->throws(\InvalidArgumentException::class, 'large_string_behavior'); + + it('leaves strings at or under the limit alone', function (): void { + $value = str_repeat('a', 40); + + expect(resolve(Redactor::class)->redact($value, 'long'))->toBe($value); + }); +}); diff --git a/tests/Feature/RedactorObjectHandlingTest.php b/tests/Feature/RedactorObjectHandlingTest.php index 2deb5cb..ebc5a8a 100644 --- a/tests/Feature/RedactorObjectHandlingTest.php +++ b/tests/Feature/RedactorObjectHandlingTest.php @@ -4,7 +4,15 @@ namespace Tests\Feature; +use Kirschbaum\Redactor\RedactionContext; use Kirschbaum\Redactor\Redactor; +use Kirschbaum\Redactor\RedactorConfig; +use Kirschbaum\Redactor\Strategies\BlockedKeysStrategy; +use Kirschbaum\Redactor\Strategies\LargeObjectStrategy; +use Kirschbaum\Redactor\Strategies\LargeStringStrategy; +use Kirschbaum\Redactor\Strategies\RegexPatternsStrategy; +use Kirschbaum\Redactor\Strategies\SafeKeysStrategy; +use Kirschbaum\Redactor\Strategies\ShannonEntropyStrategy; // Test object with toArray method for testing array conversion class TestObjectWithToArray @@ -21,19 +29,19 @@ public function toArray(): array } } -describe('Redactor Large Object Tests', function () { - beforeEach(function () { +describe('Redactor Large Object Tests', function (): void { + beforeEach(function (): void { // Set up profile-based configuration for large object tests config()->set('redactor.default_profile', 'default'); config()->set('redactor.profiles.default', [ 'enabled' => true, 'strategies' => [ - \Kirschbaum\Redactor\Strategies\SafeKeysStrategy::class, - \Kirschbaum\Redactor\Strategies\BlockedKeysStrategy::class, - \Kirschbaum\Redactor\Strategies\LargeObjectStrategy::class, - \Kirschbaum\Redactor\Strategies\LargeStringStrategy::class, - \Kirschbaum\Redactor\Strategies\RegexPatternsStrategy::class, - \Kirschbaum\Redactor\Strategies\ShannonEntropyStrategy::class, + SafeKeysStrategy::class, + BlockedKeysStrategy::class, + LargeObjectStrategy::class, + LargeStringStrategy::class, + RegexPatternsStrategy::class, + ShannonEntropyStrategy::class, ], 'safe_keys' => [], 'blocked_keys' => [], @@ -54,7 +62,7 @@ public function toArray(): array ]); }); - it('redacts large arrays based on size', function () { + it('redacts large arrays based on size', function (): void { $redactor = new Redactor; $smallArray = ['a' => 1, 'b' => 2]; @@ -74,7 +82,7 @@ public function toArray(): array ->and($result['_redacted'])->toBeTrue(); }); - it('redacts large arrays when provided as top-level input', function () { + it('redacts large arrays when provided as top-level input', function (): void { $redactor = new Redactor; // Create a large array as the primary input (not nested within another structure) @@ -91,7 +99,7 @@ public function toArray(): array ->and($result['_redacted'])->toBeTrue(); }); - it('redacts large objects based on property count', function () { + it('redacts large objects based on property count', function (): void { $redactor = new Redactor; // Create an object with many properties @@ -110,7 +118,7 @@ public function toArray(): array ->and($result['_redacted'])->toBeTrue(); }); - it('skips large object redaction when feature is disabled', function () { + it('skips large object redaction when feature is disabled', function (): void { // Update the profile config for this test config()->set('redactor.profiles.default.redact_large_objects', false); @@ -141,7 +149,7 @@ public function toArray(): array ->and($result)->not->toHaveKey('_redacted'); // No redaction occurred }); - it('handles large objects that exceed size limits during property counting', function () { + it('handles large objects that exceed size limits during property counting', function (): void { config()->set('redactor.profiles.default.max_object_size', 1); $redactor = new Redactor; @@ -160,7 +168,7 @@ public function toArray(): array expect($message)->toContain('stdClass'); }); - it('handles large objects detected via JSON encoding when toArray is unavailable', function () { + it('handles large objects detected via JSON encoding when toArray is unavailable', function (): void { config()->set('redactor.profiles.default.max_object_size', 2); $redactor = new Redactor; @@ -185,19 +193,19 @@ public function toArray(): array }); -describe('Redactor String Length Tests', function () { - beforeEach(function () { +describe('Redactor String Length Tests', function (): void { + beforeEach(function (): void { // Set up profile-based configuration for string length tests config()->set('redactor.default_profile', 'default'); config()->set('redactor.profiles.default', [ 'enabled' => true, 'strategies' => [ - \Kirschbaum\Redactor\Strategies\SafeKeysStrategy::class, - \Kirschbaum\Redactor\Strategies\BlockedKeysStrategy::class, - \Kirschbaum\Redactor\Strategies\LargeObjectStrategy::class, - \Kirschbaum\Redactor\Strategies\LargeStringStrategy::class, - \Kirschbaum\Redactor\Strategies\RegexPatternsStrategy::class, - \Kirschbaum\Redactor\Strategies\ShannonEntropyStrategy::class, + SafeKeysStrategy::class, + BlockedKeysStrategy::class, + LargeObjectStrategy::class, + LargeStringStrategy::class, + RegexPatternsStrategy::class, + ShannonEntropyStrategy::class, ], 'safe_keys' => [], 'blocked_keys' => [], @@ -218,7 +226,7 @@ public function toArray(): array ]); }); - it('redacts strings that exceed the maximum value length', function () { + it('redacts strings that exceed the maximum value length', function (): void { $redactor = new Redactor; $shortString = 'This is a short string'; @@ -233,13 +241,13 @@ public function toArray(): array expect($result['short'])->toBe($shortString) ->and($result['long'])->toContain('[REDACTED]') - ->and($result['long'])->toContain('(String with') + ->and($result['long'])->toContain('(String truncated:') ->and($result['_redacted'])->toBeTrue(); }); - it('handles string length redaction when value length is null (no limit)', function () { + it('handles string length redaction when value length is null (no limit)', function (): void { // Update the profile config to remove length limit - config()->set('redactor.profiles.default.max_value_length', null); + config()->set('redactor.profiles.default.max_value_length'); $redactor = new Redactor; @@ -252,19 +260,19 @@ public function toArray(): array }); }); -describe('Redactor Object Handling Tests', function () { - beforeEach(function () { +describe('Redactor Object Handling Tests', function (): void { + beforeEach(function (): void { // Set up profile-based configuration for object handling tests config()->set('redactor.default_profile', 'default'); config()->set('redactor.profiles.default', [ 'enabled' => true, 'strategies' => [ - \Kirschbaum\Redactor\Strategies\SafeKeysStrategy::class, - \Kirschbaum\Redactor\Strategies\BlockedKeysStrategy::class, - \Kirschbaum\Redactor\Strategies\LargeObjectStrategy::class, - \Kirschbaum\Redactor\Strategies\LargeStringStrategy::class, - \Kirschbaum\Redactor\Strategies\RegexPatternsStrategy::class, - \Kirschbaum\Redactor\Strategies\ShannonEntropyStrategy::class, + SafeKeysStrategy::class, + BlockedKeysStrategy::class, + LargeObjectStrategy::class, + LargeStringStrategy::class, + RegexPatternsStrategy::class, + ShannonEntropyStrategy::class, ], 'safe_keys' => [], 'blocked_keys' => ['password'], @@ -285,7 +293,7 @@ public function toArray(): array ]); }); - it('redacts objects with toArray method', function () { + it('redacts objects with toArray method', function (): void { $redactor = new Redactor; $object = new class @@ -309,7 +317,7 @@ public function toArray(): array ->and($result['_redacted'])->toBeTrue(); }); - it('redacts objects via JSON serialization when toArray is not available', function () { + it('redacts objects via JSON serialization when toArray is not available', function (): void { $redactor = new Redactor; $object = new \stdClass; @@ -326,7 +334,7 @@ public function toArray(): array ->and($result['_redacted'])->toBeTrue(); }); - it('handles objects with blocked and safe keys correctly', function () { + it('handles objects with blocked and safe keys correctly', function (): void { // Update the profile config for this test config()->set('redactor.profiles.default.blocked_keys', ['password', 'secret']); config()->set('redactor.profiles.default.safe_keys', ['id']); @@ -349,12 +357,12 @@ public function toArray(): array ->and($result['_redacted'])->toBeTrue(); }); - it('handles object with toArray method that throws an exception', function () { + it('handles object with toArray method that throws an exception', function (): void { $redactor = new Redactor; $objectWithBadToArray = new class { - public function toArray() + public function toArray(): never { throw new \Exception('toArray failed'); } @@ -366,12 +374,12 @@ public function toArray() expect($result)->toBeArray(); // Should still be processed as an array }); - it('handles object with toArray method that returns non-array', function () { + it('handles object with toArray method that returns non-array', function (): void { $redactor = new Redactor; $objectWithBadToArray = new class { - public function toArray() + public function toArray(): string { return 'not_an_array'; // Invalid return type } @@ -383,7 +391,7 @@ public function toArray() expect($result)->toBeArray(); }); - it('handles object with circular reference via JSON encoding', function () { + it('handles object with circular reference via JSON encoding', function (): void { $redactor = new Redactor; // Create circular reference object that can't be JSON serialized @@ -397,7 +405,7 @@ public function toArray() expect($result)->toBe($object); // Should preserve the original object }); - it('processes JsonSerializable objects correctly', function () { + it('processes JsonSerializable objects correctly', function (): void { $redactor = new Redactor; $jsonObject = new class implements \JsonSerializable @@ -421,7 +429,7 @@ public function jsonSerialize(): array ->and($result['_redacted'])->toBeTrue(); }); - it('handles objects that become problematic during toArray conversion', function () { + it('handles objects that become problematic during toArray conversion', function (): void { $redactor = new Redactor; // Create an object that has a toArray method but creates issues during conversion @@ -457,19 +465,19 @@ public function toArray(): array }); -describe('Redactor Non-Redactable Object Behavior Tests', function () { - beforeEach(function () { +describe('Redactor Non-Redactable Object Behavior Tests', function (): void { + beforeEach(function (): void { // Set up profile-based configuration for non-redactable object behavior tests config()->set('redactor.default_profile', 'default'); config()->set('redactor.profiles.default', [ 'enabled' => true, 'strategies' => [ - \Kirschbaum\Redactor\Strategies\SafeKeysStrategy::class, - \Kirschbaum\Redactor\Strategies\BlockedKeysStrategy::class, - \Kirschbaum\Redactor\Strategies\LargeObjectStrategy::class, - \Kirschbaum\Redactor\Strategies\LargeStringStrategy::class, - \Kirschbaum\Redactor\Strategies\RegexPatternsStrategy::class, - \Kirschbaum\Redactor\Strategies\ShannonEntropyStrategy::class, + SafeKeysStrategy::class, + BlockedKeysStrategy::class, + LargeObjectStrategy::class, + LargeStringStrategy::class, + RegexPatternsStrategy::class, + ShannonEntropyStrategy::class, ], 'safe_keys' => [], 'blocked_keys' => [], @@ -490,7 +498,7 @@ public function toArray(): array ]); }); - it('preserves non-redactable objects when behavior is set to preserve', function () { + it('preserves non-redactable objects when behavior is set to preserve', function (): void { $redactor = new Redactor; // Create circular reference object that can't be JSON serialized @@ -502,7 +510,7 @@ public function toArray(): array expect($result)->toBe($object); // Should preserve original object }); - it('removes non-redactable objects when behavior is set to remove', function () { + it('removes non-redactable objects when behavior is set to remove', function (): void { // Update the profile config for this test config()->set('redactor.profiles.default.non_redactable_object_behavior', 'remove'); @@ -517,7 +525,7 @@ public function toArray(): array expect($result)->toBe('__REDACTOR_REMOVE_OBJECT__'); }); - it('replaces non-redactable objects with empty array when behavior is set to empty_array', function () { + it('replaces non-redactable objects with empty array when behavior is set to empty_array', function (): void { // Update the profile config for this test config()->set('redactor.profiles.default.non_redactable_object_behavior', 'empty_array'); @@ -534,7 +542,7 @@ public function toArray(): array ->and($result['_redacted'])->toBeTrue(); }); - it('redacts non-redactable objects with replacement text when behavior is set to redact', function () { + it('redacts non-redactable objects with replacement text when behavior is set to redact', function (): void { // Update the profile config for this test config()->set('redactor.profiles.default.non_redactable_object_behavior', 'redact'); @@ -551,7 +559,7 @@ public function toArray(): array ->and($result)->toContain('stdClass'); }); - it('handles complex objects with nested non-redactable content when behavior is redact', function () { + it('handles complex objects with nested non-redactable content when behavior is redact', function (): void { // Update the profile config for this test config()->set('redactor.profiles.default.non_redactable_object_behavior', 'redact'); @@ -583,7 +591,7 @@ public function toArray(): array ->and($result['_redacted'])->toBeTrue(); }); - it('handles JsonSerializable objects that return invalid JSON when behavior is redact', function () { + it('handles JsonSerializable objects that return invalid JSON when behavior is redact', function (): void { // Update the profile config for this test config()->set('redactor.profiles.default.non_redactable_object_behavior', 'redact'); @@ -605,7 +613,7 @@ public function jsonSerialize(): string ->and($result)->toContain('[REDACTED]'); }); - it('tracks redacted keys when non-redactable object behavior is remove', function () { + it('tracks redacted keys when non-redactable object behavior is remove', function (): void { // Update the profile config for this test config()->set('redactor.profiles.default.track_redacted_keys', true); config()->set('redactor.profiles.default.non_redactable_object_behavior', 'remove'); @@ -635,7 +643,7 @@ public function jsonSerialize(): string ->and($result['_redacted_keys'])->toContain('password'); }); - it('handles non-redactable objects within arrays when behavior is empty_array', function () { + it('handles non-redactable objects within arrays when behavior is empty_array', function (): void { // Update the profile config for this test config()->set('redactor.profiles.default.non_redactable_object_behavior', 'empty_array'); @@ -660,7 +668,7 @@ public function jsonSerialize(): string ->and($result['_redacted'])->toBeTrue(); }); - it('preserves non-redactable objects by default when behavior is preserve', function () { + it('preserves non-redactable objects by default when behavior is preserve', function (): void { // Update the profile config for this test config()->set('redactor.profiles.default.non_redactable_object_behavior', 'preserve'); @@ -685,7 +693,7 @@ public function __construct() ->and($result['problematic'])->toBe($problematic); // Should preserve original }); - it('handles deeply nested non-redactable objects when behavior is preserve', function () { + it('handles deeply nested non-redactable objects when behavior is preserve', function (): void { // Update the profile config for this test config()->set('redactor.profiles.default.non_redactable_object_behavior', 'preserve'); @@ -710,7 +718,7 @@ public function __construct() ->and($result['level1']['level2']['problematic'])->toBe($circular); // Should preserve }); - it('handles objects that JSON decode to non-array values', function () { + it('handles objects that JSON decode to non-array values', function (): void { // Test when JSON decode doesn't return an array $problematicObject = new class implements \JsonSerializable { @@ -727,12 +735,12 @@ public function jsonSerialize(): string expect($result)->toBe($problematicObject); }); - it('handles LargeObjectStrategy with scalar values passed directly to handle method', function () { + it('handles LargeObjectStrategy with scalar values passed directly to handle method', function (): void { // Test the final return $value; line in LargeObjectStrategy that can only be reached // by calling handle directly with a scalar value (shouldHandle would never allow this) - $strategy = new \Kirschbaum\Redactor\Strategies\LargeObjectStrategy; - $config = \Kirschbaum\Redactor\RedactorConfig::fromConfig('default'); - $context = new \Kirschbaum\Redactor\RedactionContext($config); + $strategy = new LargeObjectStrategy; + $config = RedactorConfig::fromConfig('default'); + $context = new RedactionContext($config); // Call handle directly with scalar values expect($strategy->handle('test string', 'test_key', $context))->toBe('test string'); @@ -742,15 +750,15 @@ public function jsonSerialize(): string expect($strategy->handle(null, 'test_key', $context))->toBe(null); }); - it('handles LargeObjectStrategy toArray exception in handle method', function () { + it('handles LargeObjectStrategy toArray exception in handle method', function (): void { // Test exception handling in toArray during handle method - $strategy = new \Kirschbaum\Redactor\Strategies\LargeObjectStrategy; - $config = \Kirschbaum\Redactor\RedactorConfig::fromConfig('default'); - $context = new \Kirschbaum\Redactor\RedactionContext($config); + $strategy = new LargeObjectStrategy; + $config = RedactorConfig::fromConfig('default'); + $context = new RedactionContext($config); $problematicObject = new class { - public function toArray() + public function toArray(): never { throw new \Exception('toArray failed in handle'); } @@ -764,11 +772,11 @@ public function toArray() expect($result['_large_object_redacted'])->toContain('large number of'); }); - it('handles LargeObjectStrategy JSON encoding exception in handle method', function () { + it('handles LargeObjectStrategy JSON encoding exception in handle method', function (): void { // Test JSON encoding exception handling in handle method else branch - $strategy = new \Kirschbaum\Redactor\Strategies\LargeObjectStrategy; - $config = \Kirschbaum\Redactor\RedactorConfig::fromConfig('default'); - $context = new \Kirschbaum\Redactor\RedactionContext($config); + $strategy = new LargeObjectStrategy; + $config = RedactorConfig::fromConfig('default'); + $context = new RedactionContext($config); // Create an object without toArray method that will fail JSON encoding $problematicObject = new class diff --git a/tests/Feature/RedactorOpaqueObjectTest.php b/tests/Feature/RedactorOpaqueObjectTest.php new file mode 100644 index 0000000..472e365 --- /dev/null +++ b/tests/Feature/RedactorOpaqueObjectTest.php @@ -0,0 +1,102 @@ + $exception])); + + expect($result->context['exception'])->toBe($exception); + + $rendered = (new LineFormatter(includeStacktraces: true))->format($result); + + expect($rendered)->toContain('RuntimeException') + ->and($rendered)->toContain('db down') + ->and($rendered)->toContain('[stacktrace]'); + }); + + it('passes dates, enums and closures through untouched', function (): void { + $when = Date::parse('2026-09-13 10:00:00'); + $closure = fn (): int => 1; + $zone = new \DateTimeZone('UTC'); + + $result = resolve(Redactor::class)->redact([ + 'when' => $when, + 'status' => OpaqueStatus::Active, + 'callback' => $closure, + 'zone' => $zone, + ]); + + expect($result['when'])->toBe($when) + ->and($result['status'])->toBe(OpaqueStatus::Active) + ->and($result['callback'])->toBe($closure) + ->and($result['zone'])->toBe($zone) + ->and($result)->not->toHaveKey('_redacted'); + }); + + it('still lets a key rule win over an opaque value', function (): void { + $result = resolve(Redactor::class)->redact([ + 'secret' => OpaqueStatus::Active, + 'password' => Date::now(), + ]); + + expect($result['secret'])->toBe('[REDACTED]') + ->and($result['password'])->toBe('[REDACTED]'); + }); + + it('preserves an opaque object nested inside a structure that is otherwise redacted', function (): void { + $exception = new \LogicException('nested'); + + $result = resolve(Redactor::class)->redact([ + 'user' => ['email' => 'bob@example.com', 'error' => $exception], + ]); + + expect($result['user']['email'])->toBe('[REDACTED]') + ->and($result['user']['error'])->toBe($exception); + }); + + it('raises no deprecation while walking objects', function (): void { + $previous = set_error_handler(function (int $errno, string $errstr): bool { + if (($errno & (E_DEPRECATED | E_USER_DEPRECATED)) !== 0) { + throw new \ErrorException($errstr, 0, $errno); + } + + return false; + }); + + try { + $object = new \stdClass; + $object->email = 'bob@example.com'; + $object->child = new \stdClass; + $object->child->token = 'abc'; + + $result = resolve(Redactor::class)->redact(['payload' => $object, 'other' => new \ArrayObject(['secret' => 'x'])]); + + expect($result['payload']['email'])->toBe('[REDACTED]'); + } finally { + restore_error_handler(); + } + }); +}); diff --git a/tests/Feature/RedactorOperatorTest.php b/tests/Feature/RedactorOperatorTest.php new file mode 100644 index 0000000..e448d91 --- /dev/null +++ b/tests/Feature/RedactorOperatorTest.php @@ -0,0 +1,471 @@ +get($name)->apply( + detection($value, $entity), + new OperatorContext('[REDACTED]', $options, Pseudonymizer::fromKey(testPseudonymizationKey())), + ); +} + +function pseudoProfile(array $overrides = []): array +{ + return array_merge([ + 'enabled' => true, + 'strategies' => [RegexPatternsStrategy::class], + 'safe_keys' => [], + 'blocked_keys' => [], + 'patterns' => [ + 'email' => ['pattern' => '/[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+/', 'entity' => 'email'], + ], + 'operators' => ['default' => 'redact'], + 'replacement' => '[REDACTED]', + 'mark_redacted' => false, + 'track_redacted_keys' => false, + 'non_redactable_object_behavior' => 'preserve', + 'max_value_length' => null, + 'redact_large_objects' => false, + 'max_object_size' => 100, + 'shannon_entropy' => ['enabled' => false], + 'pseudonymization' => ['enabled' => true, 'key' => testPseudonymizationKey()], + ], $overrides); +} + +describe('Operators', function (): void { + it('redacts, masks, keeps a tail and removes', function (): void { + expect(operate('redact', 'hunter2'))->toBe('[REDACTED]') + ->and(operate('mask', 'hunter2'))->toBe('*******') + ->and(operate('partial', '4111111111111111', ['keep' => 4]))->toBe('************1111') + ->and(operate('remove', 'hunter2'))->toBe(''); + }); + + it('preserves a value while still reporting it', function (): void { + expect(operate('preserve', 'hunter2'))->toBe('hunter2'); + }); + + it('names the unknown operator rather than failing silently', function (): void { + expect(fn (): Operator => (new OperatorRegistry)->get('teleport')) + ->toThrow(\InvalidArgumentException::class, 'teleport'); + }); + + it('accepts custom operators', function (): void { + $registry = new OperatorRegistry; + $registry->register('shout', new class implements Operator + { + public function apply(Detection $d, OperatorContext $c): string + { + return strtoupper($d->value); + } + }); + + expect($registry->get('shout')->apply(detection('quiet'), new OperatorContext('[R]')))->toBe('QUIET'); + }); +}); + +describe('Operator configuration shapes', function (): void { + it('accepts a bare name, a name with options, and an explicit key', function (): void { + expect(OperatorSpec::parse('partial', 'p')->name)->toBe('partial') + ->and(OperatorSpec::parse(['partial' => ['keep' => 6]], 'p')->options)->toBe(['keep' => 6]) + ->and(OperatorSpec::parse(['operator' => 'partial', 'keep' => 6], 'p')->name)->toBe('partial') + ->and(OperatorSpec::parse(['operator' => 'partial', 'keep' => 6], 'p')->options)->toBe(['keep' => 6]); + }); + + it('rejects a definition that names no operator', function (): void { + expect(fn (): OperatorSpec => OperatorSpec::parse([], 'profiles.x.operators.y')) + ->toThrow(\InvalidArgumentException::class, 'profiles.x.operators.y'); + }); +}); + +describe('Operator precedence', function (): void { + function precedenceProfile(array $patterns, array $operators): array + { + return pseudoProfile(['patterns' => $patterns, 'operators' => $operators]); + } + + it('applies operators.default to a rule that asked for nothing', function (): void { + // `mode` defaults to replace, so a rule can always produce an operator + // spec - which is not the same as having chosen one. Treating the + // default as a choice made operators.default unreachable for anything + // found by a pattern, silently. + config()->set('redactor.profiles.prec', precedenceProfile( + ['digits' => '/\d+/'], + ['default' => 'mask'], + )); + + expect(resolve(Redactor::class)->redact('a1b22c', 'prec'))->toBe('a*b**c'); + }); + + it('lets a rule that did choose a mode outrank the default', function (): void { + config()->set('redactor.profiles.prec', precedenceProfile( + ['digits' => ['pattern' => '/\d+/', 'mode' => 'remove']], + ['default' => 'mask'], + )); + + expect(resolve(Redactor::class)->redact('a1b22c', 'prec'))->toBe('abc'); + }); + + it('lets a rule with an explicit operator outrank the default', function (): void { + config()->set('redactor.profiles.prec', precedenceProfile( + ['digits' => ['pattern' => '/\d+/', 'operator' => 'remove']], + ['default' => 'mask'], + )); + + expect(resolve(Redactor::class)->redact('a1b22c', 'prec'))->toBe('abc'); + }); + + it('lets the entity outrank both the rule and the default', function (): void { + config()->set('redactor.profiles.prec', precedenceProfile( + ['digits' => ['pattern' => '/\d+/', 'entity' => 'num', 'mode' => 'remove']], + ['default' => 'mask', 'num' => 'redact'], + )); + + expect(resolve(Redactor::class)->redact('a1b22c', 'prec'))->toBe('a[REDACTED]b[REDACTED]c'); + }); + + it('falls back to redaction when no operator is configured anywhere', function (): void { + config()->set('redactor.profiles.prec', precedenceProfile(['digits' => '/\d+/'], [])); + + expect(resolve(Redactor::class)->redact('a1b22c', 'prec'))->toBe('a[REDACTED]b[REDACTED]c'); + }); +}); + +describe('Deterministic pseudonymization', function (): void { + it('maps the same value to the same surrogate every time', function (): void { + $a = operate('surrogate', 'alice@customer.com', [], 'email'); + $b = operate('surrogate', 'alice@customer.com', [], 'email'); + + expect($a)->toBe($b); + }); + + it('maps different values to different surrogates', function (): void { + expect(operate('surrogate', 'alice@customer.com', [], 'email')) + ->not->toBe(operate('surrogate', 'bob@customer.com', [], 'email')); + }); + + it('normalises case and whitespace so the mapping stays joinable', function (): void { + // The same person written two ways has to land on the same surrogate, + // or grouping by it silently double-counts. + expect(operate('surrogate', ' Alice@Customer.COM ', [], 'email')) + ->toBe(operate('surrogate', 'alice@customer.com', [], 'email')); + }); + + it('produces different surrogates under a different key', function (): void { + $withKeyA = (new OperatorRegistry)->get('surrogate')->apply( + detection('alice@customer.com', 'email'), + new OperatorContext('[R]', [], Pseudonymizer::fromKey(testPseudonymizationKey())), + ); + + $withKeyB = (new OperatorRegistry)->get('surrogate')->apply( + detection('alice@customer.com', 'email'), + new OperatorContext('[R]', [], Pseudonymizer::fromKey('a-completely-different-key-also-long-enough')), + ); + + expect($withKeyA)->not->toBe($withKeyB); + }); + + it('falls back to plain redaction when no key is available', function (): void { + // An unkeyed surrogate would look joinable and silently not be. + $result = (new OperatorRegistry)->get('surrogate')->apply( + detection('alice@customer.com', 'email'), + new OperatorContext('[REDACTED]', []), + ); + + expect($result)->toBe('[REDACTED]'); + }); + + it('rejects a key too short to be worth having', function (): void { + expect(fn (): Pseudonymizer => Pseudonymizer::fromKey('short')) + ->toThrow(\RuntimeException::class, 'at least'); + }); + + it('derives a key from APP_KEY without using it directly', function (): void { + $derived = Pseudonymizer::derivedFrom('base64:'.base64_encode(str_repeat('k', 32))); + $direct = Pseudonymizer::fromKey(str_repeat('k', 32)); + + expect($derived->digest('email', 'a@b.com'))->not->toBe($direct->digest('email', 'a@b.com')); + }); + + it('emits a stable labelled token in hash mode', function (): void { + $token = operate('hash', 'alice@customer.com', [], 'email'); + + expect($token)->toStartWith('[email:') + ->and($token)->toEndWith(']') + ->and($token)->toBe(operate('hash', 'alice@customer.com', [], 'email')) + ->and($token)->not->toContain('alice'); + }); +}); + +describe('Format-preserving surrogates', function (): void { + it('keeps an email parseable and its domain intact', function (): void { + $result = operate('surrogate', 'alice@customer.com', ['preserve_domain' => true], 'email'); + + expect($result)->toEndWith('@customer.com') + ->and($result)->not->toContain('alice') + ->and(filter_var($result, FILTER_VALIDATE_EMAIL))->not->toBeFalse(); + }); + + it('replaces the domain with a guaranteed-unroutable one when asked', function (): void { + // RFC 2606 reserves .invalid, so a surrogate that escapes into a mail + // queue bounces rather than reaching a stranger. + expect(operate('surrogate', 'alice@customer.com', ['preserve_domain' => false], 'email')) + ->toEndWith('@example.invalid'); + }); + + it('keeps a card Luhn-valid, same length, same grouping', function (): void { + $result = operate('surrogate', '4111 1111 1111 1111', ['preserve_bin' => 6], 'credit_card'); + + expect($result)->not->toBe('4111 1111 1111 1111') + ->and(strlen($result))->toBe(19) + ->and($result)->toStartWith('4111 11') + ->and(Validator::luhn($result))->toBeTrue() + ->and(preg_match('/^\d{4} \d{4} \d{4} \d{4}$/', $result))->toBe(1); + }); + + it('preserves character classes and separators for anything else', function (): void { + $result = (new CharacterClassSurrogate)->generate( + 'sk_live_4eC39HqLyj', + Pseudonymizer::fromKey(testPseudonymizationKey())->random('generic', 'sk_live_4eC39HqLyj'), + ['preserve_prefix' => 8], + ); + + expect($result)->toStartWith('sk_live_') + ->and(strlen($result))->toBe(strlen('sk_live_4eC39HqLyj')) + ->and($result)->not->toBe('sk_live_4eC39HqLyj'); + + // Same shape, character class for character class. + $original = 'sk_live_4eC39HqLyj'; + for ($i = 0; $i < strlen($original); $i++) { + expect(ctype_digit($result[$i]))->toBe(ctype_digit($original[$i])) + ->and(ctype_upper($result[$i]))->toBe(ctype_upper($original[$i])) + ->and(ctype_lower($result[$i]))->toBe(ctype_lower($original[$i])); + } + }); + + it('preserves digit positions and punctuation in a structured value', function (): void { + $original = '2024-01-15T09:31:00Z'; + + $result = (new CharacterClassSurrogate)->generate( + $original, + Pseudonymizer::fromKey(testPseudonymizationKey())->random('generic', $original), + ); + + // The contract is character classes, not semantics: it does not know + // this is a timestamp, so the 'T' and 'Z' are letters like any other + // and get replaced. Digit positions, length and punctuation survive. + expect(preg_match('/^\d{4}-\d{2}-\d{2}[A-Z]\d{2}:\d{2}:\d{2}[A-Z]$/', $result))->toBe(1) + ->and($result)->not->toBe($original) + ->and(strlen($result))->toBe(strlen($original)); + }); + + it('picks the most specific generator for the entity', function (): void { + expect((new EmailSurrogate)->supports('email', 'a@b.com'))->toBeTrue() + ->and((new EmailSurrogate)->supports('generic', 'no-at-sign'))->toBeFalse() + ->and((new CreditCardSurrogate)->supports('credit_card', '4111111111111111'))->toBeTrue() + ->and((new CreditCardSurrogate)->supports('generic', 'abc'))->toBeFalse() + ->and((new CharacterClassSurrogate)->supports('anything', 'at all'))->toBeTrue(); + }); +}); + +describe('Pseudonymization end to end', function (): void { + it('keeps a log line joinable through the redactor', function (): void { + config()->set('redactor.profiles.pseudo', pseudoProfile([ + 'operators' => ['default' => 'redact', 'email' => ['surrogate' => ['preserve_domain' => true]]], + ])); + + $first = resolve(Redactor::class)->redact('login from alice@customer.com ok', 'pseudo'); + $second = resolve(Redactor::class)->redact('logout for alice@customer.com ok', 'pseudo'); + + preg_match('/(\S+@customer\.com)/', $first, $a); + preg_match('/(\S+@customer\.com)/', $second, $b); + + expect($a[1] ?? 'x')->toBe($b[1] ?? 'y') + ->and($first)->not->toContain('alice') + ->and($first)->toStartWith('login from '); + }); + + it('lets one entity be pseudonymised while another is redacted', function (): void { + config()->set('redactor.profiles.pseudo', pseudoProfile([ + 'patterns' => [ + 'email' => ['pattern' => '/[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+/', 'entity' => 'email'], + 'card' => ['pattern' => '/\b\d{16}\b/', 'entity' => 'credit_card'], + ], + 'operators' => [ + 'default' => 'redact', + 'email' => 'surrogate', + 'credit_card' => 'redact', + ], + ])); + + $result = resolve(Redactor::class)->redact('a@b.com paid with 4111111111111111', 'pseudo'); + + expect($result)->toContain('@b.com') + ->and($result)->not->toContain('a@b.com') + ->and($result)->toContain('[REDACTED]'); + }); + + it('honours the shipped observability profile', function (): void { + config()->set('redactor.pseudonymization', ['enabled' => true, 'key' => testPseudonymizationKey()]); + + $result = resolve(Redactor::class)->redact([ + 'message' => 'checkout by alice@customer.com from 203.0.113.9', + 'trace_id' => 'abc-123', + ], 'observability'); + + expect($result['trace_id'])->toBe('abc-123') + ->and($result['message'])->toStartWith('checkout by ') + ->and($result['message'])->not->toContain('alice@customer.com') + ->and($result['message'])->toContain('@customer.com') + ->and($result['message'])->not->toContain('203.0.113.9'); + }); + + it('degrades to redaction rather than emitting an unkeyed surrogate', function (): void { + config()->set('redactor.profiles.pseudo', pseudoProfile([ + 'operators' => ['default' => 'redact', 'email' => 'surrogate'], + 'pseudonymization' => ['enabled' => false], + ])); + + expect(resolve(Redactor::class)->redact('mail a@b.com now', 'pseudo')) + ->toBe('mail [REDACTED] now'); + }); +}); + +describe('Operator specs in their other shapes', function (): void { + it('rejects a list that names no operator', function (): void { + expect(fn (): OperatorSpec => OperatorSpec::parse(['partial'], 'p')) + ->toThrow(\InvalidArgumentException::class, 'must name an operator'); + }); + + it('hands back a spec that is already parsed, so config built in PHP can pass one', function (): void { + $spec = new OperatorSpec('mask', ['mask_character' => '#']); + + config()->set('redactor.profiles.pseudo', pseudoProfile([ + 'patterns' => ['email' => ['pattern' => '/[a-z]+@[a-z.]+/', 'entity' => 'email', 'operator' => $spec]], + ])); + + expect(OperatorSpec::parse($spec, 'p'))->toBe($spec) + ->and(resolve(Redactor::class)->redact('mail bob@example.com', 'pseudo'))->toBe('mail ###############'); + }); + + it('falls back to the replacement when a profile names an operator nobody registered', function (): void { + config()->set('redactor.profiles.pseudo', pseudoProfile(['operators' => ['default' => 'teleport']])); + + $result = resolve(Redactor::class)->inspect('mail bob@example.com', 'pseudo'); + + expect($result->value)->toBe('mail [REDACTED]') + ->and($result->wasRedacted)->toBeTrue(); + }); +}); + +describe('Operator context', function (): void { + it('redacts plainly in hash mode when there is no key, rather than emit an unkeyed token', function (): void { + $hashed = (new OperatorRegistry)->get('hash')->apply(detection('hunter2'), new OperatorContext('[REDACTED]')); + + expect($hashed)->toBe('[REDACTED]'); + }); + + it('resolves the pseudonymizer once, however many times it is asked', function (): void { + $resolved = 0; + $context = new OperatorContext('[REDACTED]', [], function () use (&$resolved): Pseudonymizer { + $resolved++; + + return Pseudonymizer::fromKey(testPseudonymizationKey()); + }); + + $first = $context->pseudonymizer(); + + expect($context->pseudonymizer())->toBe($first) + ->and($resolved)->toBe(1); + }); +}); + +describe('Pseudonymization key fallbacks', function (): void { + beforeEach(fn () => config()->set('redactor.profiles.pseudo', pseudoProfile([ + 'operators' => ['default' => 'surrogate'], + 'pseudonymization' => ['enabled' => true], + ]))); + + it('redacts plainly when neither a key nor APP_KEY is configured', function (): void { + config()->set('redactor.pseudonymization.key'); + config()->set('app.key', ''); + + expect(resolve(Redactor::class)->redact('mail bob@example.com', 'pseudo'))->toBe('mail [REDACTED]'); + }); + + it('redacts plainly when the configured key is unusable, rather than throwing mid-log-line', function (): void { + config()->set('redactor.pseudonymization.key', 'short'); + + expect(resolve(Redactor::class)->redact('mail bob@example.com', 'pseudo'))->toBe('mail [REDACTED]'); + }); +}); + +describe('Surrogate generators at their edges', function (): void { + it('leaves a card with fewer than two digits alone, since there is nothing to keep Luhn-valid', function (): void { + expect((new CreditCardSurrogate)->generate('7', new DeterministicRandom('k', 's')))->toBe('7'); + }); + + it('invents a whole address when an email entity carries no @', function (): void { + expect((new EmailSurrogate)->generate('not-an-address', new DeterministicRandom('k', 's'))) + ->toMatch('/^u_[a-z0-9]{6}@example\.invalid$/'); + }); + + it('lets a registered generator claim a value ahead of the built-ins, and shapes the rest', function (): void { + $factory = new SurrogateFactory; + $factory->register(new class implements SurrogateGenerator + { + public function supports(string $entity, string $value): bool + { + return $entity === 'email'; + } + + public function generate(string $value, DeterministicRandom $random, array $options = []): string + { + return 'claimed'; + } + }); + + expect($factory->generate('email', 'bob@example.com', new DeterministicRandom('k', 's')))->toBe('claimed') + ->and($factory->generate('policy', 'AB-1234', new DeterministicRandom('k', 's')))->toMatch('/^[A-Z]{2}-\d{4}$/'); + }); +}); + +describe('Deterministic random', function (): void { + it('answers zero for a bound of one without drawing a byte', function (): void { + $random = new DeterministicRandom('k', 's'); + + expect($random->below(1))->toBe(0) + ->and($random->below(0))->toBe(0) + ->and($random->token(3, 'a'))->toBe('aaa'); + }); +}); diff --git a/tests/Feature/RedactorPathRulesTest.php b/tests/Feature/RedactorPathRulesTest.php new file mode 100644 index 0000000..6b66df5 --- /dev/null +++ b/tests/Feature/RedactorPathRulesTest.php @@ -0,0 +1,339 @@ + true, + 'strategies' => [BlockedKeysStrategy::class, RegexPatternsStrategy::class], + 'safe_keys' => [], + 'blocked_keys' => [], + 'patterns' => [], + 'paths' => $paths, + 'operators' => ['default' => 'redact'], + 'replacement' => '[REDACTED]', + 'mark_redacted' => false, + 'track_redacted_keys' => false, + 'non_redactable_object_behavior' => 'preserve', + 'max_value_length' => null, + 'redact_large_objects' => false, + 'max_object_size' => 100, + 'shannon_entropy' => ['enabled' => false], + 'pseudonymization' => ['enabled' => true, 'key' => testPseudonymizationKey()], + ], $overrides); +} + +function redactPath(array $paths, array $payload, array $overrides = []): array +{ + config()->set('redactor.profiles.paths', pathProfile($paths, $overrides)); + + $result = resolve(Redactor::class)->redact($payload, 'paths'); + + return is_array($result) ? $result : []; +} + +describe('Path matching', function (): void { + it('matches an exact path and nothing else', function (): void { + $result = redactPath( + ['request.headers.authorization' => 'redact'], + [ + 'request' => ['headers' => ['authorization' => 'Bearer abc', 'accept' => 'json']], + 'authorization' => 'top level, different place', + ], + ); + + expect($result['request']['headers']['authorization'])->toBe('[REDACTED]') + ->and($result['request']['headers']['accept'])->toBe('json') + // A key rule for "authorization" would have caught this too. A path + // rule is precise about where, which is the point. + ->and($result['authorization'])->toBe('top level, different place'); + }); + + it('matches a single level with *', function (): void { + $result = redactPath( + ['user.*.email' => 'redact'], + ['user' => [ + 'primary' => ['email' => 'a@b.com'], + 'billing' => ['email' => 'c@d.com'], + 'deep' => ['nested' => ['email' => 'e@f.com']], + ]], + ); + + expect($result['user']['primary']['email'])->toBe('[REDACTED]') + ->and($result['user']['billing']['email'])->toBe('[REDACTED]') + // One level, not any level. + ->and($result['user']['deep']['nested']['email'])->toBe('e@f.com'); + }); + + it('matches any depth with **', function (): void { + $result = redactPath( + ['**.password' => 'redact'], + [ + 'password' => 'top', + 'a' => ['password' => 'one deep'], + 'b' => ['c' => ['d' => ['password' => 'four deep']]], + 'keep' => 'visible', + ], + ); + + expect($result['password'])->toBe('[REDACTED]') + ->and($result['a']['password'])->toBe('[REDACTED]') + ->and($result['b']['c']['d']['password'])->toBe('[REDACTED]') + ->and($result['keep'])->toBe('visible'); + }); + + it('lets ** match zero segments', function (): void { + $result = redactPath( + ['a.**.secret' => 'redact'], + ['a' => ['secret' => 'immediately below a']], + ); + + expect($result['a']['secret'])->toBe('[REDACTED]'); + }); + + it('walks through lists with either spelling', function (): void { + $bracket = redactPath(['users[*].token' => 'redact'], [ + 'users' => [['token' => 'one'], ['token' => 'two']], + ]); + + $dotted = redactPath(['users.*.token' => 'redact'], [ + 'users' => [['token' => 'one'], ['token' => 'two']], + ]); + + expect($bracket['users'][0]['token'])->toBe('[REDACTED]') + ->and($bracket['users'][1]['token'])->toBe('[REDACTED]') + ->and($dotted)->toBe($bracket); + }); + + it('matches path segments case-insensitively', function (): void { + $result = redactPath( + ['request.headers.authorization' => 'redact'], + ['Request' => ['Headers' => ['Authorization' => 'Bearer abc']]], + ); + + expect($result['Request']['Headers']['Authorization'])->toBe('[REDACTED]'); + }); + + it('leaves a payload with no matching path completely alone', function (): void { + $payload = ['a' => ['b' => 'value'], 'c' => 'other']; + + expect(redactPath(['x.y.z' => 'redact'], $payload))->toBe($payload); + }); +}); + +describe('Path precedence', function (): void { + it('prefers the more specific pattern regardless of declaration order', function (): void { + $specificLast = redactPath( + ['**.token' => 'redact', 'auth.token' => ['partial' => ['keep' => 4]]], + ['auth' => ['token' => 'abcdefgh']], + ); + + $specificFirst = redactPath( + ['auth.token' => ['partial' => ['keep' => 4]], '**.token' => 'redact'], + ['auth' => ['token' => 'abcdefgh']], + ); + + expect($specificLast['auth']['token'])->toBe('****efgh') + ->and($specificFirst['auth']['token'])->toBe('****efgh'); + }); + + it('beats a key rule that would otherwise fire', function (): void { + // The key rule says redact anything called token; the path rule carves + // out one location and keeps it. + $result = redactPath( + ['public.token' => 'preserve'], + ['public' => ['token' => 'not-a-secret'], 'private' => ['token' => 'secret']], + ['blocked_keys' => ['*token*']], + ); + + expect($result['public']['token'])->toBe('not-a-secret') + ->and($result['private']['token'])->toBe('[REDACTED]'); + }); + + it('stops the walk at the matched node', function (): void { + // The path names a whole subtree, so nothing beneath it is inspected. + $result = redactPath( + ['debug' => 'preserve'], + ['debug' => ['password' => 'kept', 'nested' => ['token' => 'kept too']]], + ['blocked_keys' => ['password', '*token*']], + ); + + expect($result['debug'])->toBe(['password' => 'kept', 'nested' => ['token' => 'kept too']]); + }); +}); + +describe('Path operators', function (): void { + it('supports the full operator range on a scalar', function (): void { + $payload = ['a' => ['v' => 'abcdefgh']]; + + expect(redactPath(['a.v' => 'mask'], $payload)['a']['v'])->toBe('********') + ->and(redactPath(['a.v' => ['partial' => ['keep' => 3]]], $payload)['a']['v'])->toBe('*****fgh') + ->and(redactPath(['a.v' => 'preserve'], $payload)['a']['v'])->toBe('abcdefgh'); + }); + + it('drops the key entirely under remove', function (): void { + $result = redactPath(['a.gone' => 'remove'], ['a' => ['gone' => 'x', 'kept' => 'y']]); + + expect($result['a'])->toBe(['kept' => 'y']); + }); + + it('pseudonymises at a path', function (): void { + $result = redactPath(['user.email' => 'surrogate'], ['user' => ['email' => 'alice@customer.com']]); + + expect($result['user']['email'])->toEndWith('@customer.com') + ->and($result['user']['email'])->not->toContain('alice'); + }); + + it('replaces a whole subtree when the operator has no meaning for a container', function (): void { + // Masking an array has no defensible behaviour, so the subtree is + // replaced rather than a behaviour being invented for it. + $result = redactPath(['a.b' => 'mask'], ['a' => ['b' => ['x' => 1, 'y' => 2]]]); + + expect($result['a']['b'])->toBe('[REDACTED]'); + }); + + it('reports the pattern that fired', function (): void { + config()->set('redactor.profiles.paths', pathProfile( + ['request.headers.authorization' => 'redact'], + ['track_redacted_keys' => true, 'mark_redacted' => true], + )); + + $result = resolve(Redactor::class)->inspect( + ['request' => ['headers' => ['authorization' => 'Bearer abc']]], + 'paths', + ); + + expect($result->wasRedacted)->toBeTrue() + ->and($result->findings[0]->rule)->toBe('path:request.headers.authorization'); + }); +}); + +describe('Compiled state invalidates on config change', function (): void { + it('rebuilds the trie when a path rule is added', function (): void { + $before = redactPath(['a.one' => 'redact'], ['a' => ['one' => 'x', 'two' => 'y']]); + + expect($before['a'])->toBe(['one' => '[REDACTED]', 'two' => 'y']); + + $after = redactPath(['a.one' => 'redact', 'a.two' => 'redact'], ['a' => ['one' => 'x', 'two' => 'y']]); + + expect($after['a'])->toBe(['one' => '[REDACTED]', 'two' => '[REDACTED]']); + }); + + it('rebuilds when only the operator changes', function (): void { + // Same pattern, different verb. A cache keyed on patterns alone would + // serve the old operator and the config change would silently not + // apply - the failure mode that turns a cache into a security bug. + $redacted = redactPath(['a.v' => 'redact'], ['a' => ['v' => 'abcdefgh']]); + $masked = redactPath(['a.v' => 'mask'], ['a' => ['v' => 'abcdefgh']]); + + expect($redacted['a']['v'])->toBe('[REDACTED]') + ->and($masked['a']['v'])->toBe('********'); + }); + + it('rebuilds when only an operator option changes', function (): void { + $keepFour = redactPath(['a.v' => ['partial' => ['keep' => 4]]], ['a' => ['v' => 'abcdefgh']]); + $keepTwo = redactPath(['a.v' => ['partial' => ['keep' => 2]]], ['a' => ['v' => 'abcdefgh']]); + + expect($keepFour['a']['v'])->toBe('****efgh') + ->and($keepTwo['a']['v'])->toBe('******gh'); + }); + + it('rebuilds the profile when an unrelated setting changes', function (): void { + $first = redactPath(['a.v' => 'redact'], ['a' => ['v' => 'x']]); + + expect($first['a']['v'])->toBe('[REDACTED]'); + + $second = redactPath(['a.v' => 'redact'], ['a' => ['v' => 'x']], ['replacement' => '']); + + expect($second['a']['v'])->toBe(''); + }); + + it('rebuilds when the profile is disabled', function (): void { + expect(redactPath(['a.v' => 'redact'], ['a' => ['v' => 'x']])['a']['v'])->toBe('[REDACTED]'); + + config()->set('redactor.profiles.paths', pathProfile(['a.v' => 'redact'], ['enabled' => false])); + + expect(resolve(Redactor::class)->redact(['a' => ['v' => 'x']], 'paths'))->toBe(['a' => ['v' => 'x']]); + }); +}); + +describe('Path compilation', function (): void { + it('normalises the two list spellings to the same segments', function (): void { + expect(PathPattern::parse('users[*].email')->segments) + ->toBe(PathPattern::parse('users.*.email')->segments); + }); + + it('scores literals above single wildcards above deep wildcards', function (): void { + $literal = PathPattern::parse('a.b.c')->specificity; + $single = PathPattern::parse('a.*.c')->specificity; + $deep = PathPattern::parse('a.**.c')->specificity; + + expect($literal)->toBeGreaterThan($single) + ->and($single)->toBeGreaterThan($deep); + }); + + it('accepts a purely numeric pattern, which PHP hands over as an int', function (): void { + // 'items.0' => 'redact' is a reasonable rule, and '0' => 'redact' more + // so for a list payload. PHP turns a numeric array key into an integer, + // which used to reach PathPattern::parse() and fail its string type. + $result = redactPath(['1' => 'redact'], ['zero', 'one', 'two']); + + expect($result)->toBe(['zero', '[REDACTED]', 'two']); + }); + + it('targets a list index through a longer path', function (): void { + $result = redactPath(['items.0.token' => 'redact'], [ + 'items' => [['token' => 'first'], ['token' => 'second']], + ]); + + expect($result['items'][0]['token'])->toBe('[REDACTED]') + ->and($result['items'][1]['token'])->toBe('second'); + }); + + it('rejects an empty pattern', function (): void { + expect(fn (): PathPattern => PathPattern::parse('...')) + ->toThrow(\InvalidArgumentException::class); + }); + + it('reports an empty trie as empty, and never matches', function (): void { + $trie = PathTrie::compile([]); + + expect($trie->isEmpty())->toBeTrue() + ->and($trie->cursor()->isExhausted())->toBeTrue() + ->and($trie->cursor()->descend('anything')->match())->toBeNull(); + }); + + it('exhausts the cursor once no rule can still match', function (): void { + $trie = PathTrie::compile(['a.b' => new OperatorSpec('redact')]); + + expect($trie->cursor()->descend('a')->isExhausted())->toBeFalse() + ->and($trie->cursor()->descend('z')->isExhausted())->toBeTrue(); + }); + + it('keeps a deep-wildcard cursor alive at every level', function (): void { + $trie = PathTrie::compile(['**.secret' => new OperatorSpec('redact')]); + + $cursor = $trie->cursor()->descend('a')->descend('b')->descend('c'); + + expect($cursor->isExhausted())->toBeFalse() + ->and($cursor->descend('secret')->match())->not->toBeNull(); + }); +}); + +describe('Path trie states', function (): void { + it('keeps an exhausted state set exhausted, whatever segment follows', function (): void { + $trie = PathTrie::compile(['a.b' => new OperatorSpec('redact')]); + + expect($trie->advance([], 'a'))->toBe([]); + }); +}); diff --git a/tests/Feature/RedactorPcreFailureTest.php b/tests/Feature/RedactorPcreFailureTest.php new file mode 100644 index 0000000..d13bff9 --- /dev/null +++ b/tests/Feature/RedactorPcreFailureTest.php @@ -0,0 +1,185 @@ +toBeFalse() + ->and(preg_last_error())->toBe(PREG_BAD_UTF8_ERROR); + }); + + it('treats an unevaluatable detection pattern as a match', function (): void { + config()->set('redactor.profiles.pcre', [ + 'enabled' => true, + 'strategies' => [RegexPatternsStrategy::class], + 'safe_keys' => [], + 'blocked_keys' => [], + 'patterns' => ['unevaluatable' => failingPattern()], + 'replacement' => '[REDACTED]', + 'mark_redacted' => false, + 'track_redacted_keys' => false, + 'non_redactable_object_behavior' => 'preserve', + 'max_value_length' => null, + 'redact_large_objects' => false, + 'max_object_size' => 100, + 'shannon_entropy' => ['enabled' => false], + ]); + + $result = resolve(Redactor::class)->redact(['note' => failingSubject()], 'pcre'); + + // Before: preg_match returned false, was read as "no match", and the + // value went out untouched. + expect($result['note'])->toBe('[REDACTED]'); + }); + + it('does not let an unevaluatable exclusion pattern excuse a value', function (): void { + config()->set('redactor.profiles.pcre_exclusion', [ + 'enabled' => true, + 'strategies' => [ShannonEntropyStrategy::class], + 'safe_keys' => [], + 'blocked_keys' => [], + 'patterns' => [], + 'replacement' => '[REDACTED]', + 'mark_redacted' => false, + 'track_redacted_keys' => false, + 'non_redactable_object_behavior' => 'preserve', + 'max_value_length' => null, + 'redact_large_objects' => false, + 'max_object_size' => 100, + 'shannon_entropy' => [ + 'enabled' => true, + 'threshold' => 3.0, + 'min_length' => 10, + // An exclusion pattern that cannot be evaluated against this + // subject must not be read as "excluded". + 'exclusion_patterns' => [failingPattern()], + ], + ]); + + $excluded = (new ShannonEntropyStrategy)->isCommonPattern( + failingSubject(), + RedactorConfig::fromConfig('pcre_exclusion') + ); + + expect($excluded)->toBeFalse(); + }); + + it('replaces a value the entropy tokeniser cannot even split', function (): void { + config()->set('redactor.profiles.pcre_entropy', [ + 'enabled' => true, + 'strategies' => [ShannonEntropyStrategy::class], + 'safe_keys' => [], + 'blocked_keys' => [], + 'patterns' => [], + 'replacement' => '[REDACTED]', + 'mark_redacted' => false, + 'track_redacted_keys' => false, + 'non_redactable_object_behavior' => 'preserve', + 'max_value_length' => null, + 'redact_large_objects' => false, + 'max_object_size' => 100, + 'shannon_entropy' => [ + 'enabled' => true, + 'threshold' => 3.0, + 'min_length' => 10, + 'exclusion_patterns' => [], + ], + ]); + + $result = resolve(Redactor::class)->redact( + ['note' => failingSubject()."\xfe high entropy Zx7Qm4Kd9Rb2Vn6Tp1Ws8Yc3Hf"], + 'pcre_entropy' + ); + + expect($result['note'])->toBe('[REDACTED]'); + }); + + it('treats an unevaluatable blocked-key pattern as blocking the key', function (): void { + config()->set('redactor.profiles.pcre_keys', [ + 'enabled' => true, + 'strategies' => [BlockedKeysStrategy::class], + 'safe_keys' => [], + 'blocked_keys' => ['*'.failingSubject().'*'], + 'patterns' => [], + 'replacement' => '[REDACTED]', + 'mark_redacted' => false, + 'track_redacted_keys' => false, + 'non_redactable_object_behavior' => 'preserve', + 'max_value_length' => null, + 'redact_large_objects' => false, + 'max_object_size' => 100, + 'shannon_entropy' => ['enabled' => false], + ]); + + // A *contains* pattern is compiled to str_contains, which cannot fail, + // so this asserts the safe-by-construction path rather than the + // fail-closed one. The regex branch is covered by the Pcre tests below. + $result = resolve(Redactor::class)->redact(['a'.failingSubject().'b' => 'value'], 'pcre_keys'); + + expect(array_values($result)[0])->toBe('[REDACTED]'); + }); +}); + +describe('Pcre helper', function (): void { + it('reports a normal match and non-match correctly', function (): void { + expect(Pcre::matches('/foo/', 'a foo b', onError: true))->toBeTrue() + ->and(Pcre::matches('/foo/', 'a bar b', onError: true))->toBeFalse(); + }); + + it('returns the caller-chosen answer on engine failure', function (): void { + expect(Pcre::matches(failingPattern(), failingSubject(), onError: true))->toBeTrue() + ->and(Pcre::matches(failingPattern(), failingSubject(), onError: false))->toBeFalse(); + }); + + it('returns null from replaceCallback when the engine fails', function (): void { + $out = Pcre::replaceCallback( + failingPattern(), + fn (array $m): string => '[X]', + failingSubject() + ); + + expect($out)->toBeNull(); + }); + + it('replaces normally when the engine succeeds', function (): void { + expect(Pcre::replaceCallback('/\d+/', fn (array $m): string => '#', 'a1b22c'))->toBe('a#b#c'); + }); + + it('recognises invalid patterns without emitting a PHP warning', function (): void { + expect(Pcre::isValidPattern('/valid/'))->toBeTrue() + ->and(Pcre::isValidPattern('/[unclosed/'))->toBeFalse(); + }); +}); diff --git a/tests/Feature/RedactorProcessorTest.php b/tests/Feature/RedactorProcessorTest.php new file mode 100644 index 0000000..92c6666 --- /dev/null +++ b/tests/Feature/RedactorProcessorTest.php @@ -0,0 +1,191 @@ +toBeInstanceOf(ProcessorInterface::class); + }); + + it('redacts the message and leaves the rest of the record intact', function (): void { + $processor = new RedactorProcessor(resolve(Redactor::class)); + + $result = $processor(logRecord('User bob@example.com signed in')); + + expect($result->message)->toBe('User [REDACTED] signed in') + ->and($result->channel)->toBe('testing') + ->and($result->level)->toBe(Level::Info); + }); + + it('redacts context', function (): void { + $processor = new RedactorProcessor(resolve(Redactor::class)); + + $result = $processor(logRecord('hi', ['password' => 'hunter2', 'keep' => 'visible'])); + + expect($result->context['password'])->toBe('[REDACTED]') + ->and($result->context['keep'])->toBe('visible'); + }); + + it('redacts extra, which the formatter dropped entirely', function (): void { + $processor = new RedactorProcessor(resolve(Redactor::class)); + + $result = $processor(logRecord('hi', [], ['api_token' => 'abc123', 'pid' => 42])); + + expect($result->extra['api_token'])->toBe('[REDACTED]') + ->and($result->extra['pid'])->toBe(42); + }); + + it('honours a profile override', function (): void { + config()->set('redactor.profiles.tapped', [ + 'enabled' => true, + 'strategies' => [BlockedKeysStrategy::class], + 'safe_keys' => [], + 'blocked_keys' => ['keep'], + 'patterns' => [], + 'replacement' => '', + 'mark_redacted' => false, + 'track_redacted_keys' => false, + 'non_redactable_object_behavior' => 'preserve', + 'max_value_length' => null, + 'redact_large_objects' => false, + 'max_object_size' => 100, + 'shannon_entropy' => ['enabled' => false], + ]); + + $processor = new RedactorProcessor(resolve(Redactor::class), 'tapped'); + + expect($processor(logRecord('hi', ['keep' => 'x']))->context['keep'])->toBe(''); + }); + + it('never throws when the profile is broken', function (): void { + $processor = new RedactorProcessor(resolve(Redactor::class), 'no_such_profile'); + + $result = $processor(logRecord('bob@example.com', ['password' => 'hunter2'])); + + expect($result->message)->not->toContain('bob@example.com') + ->and(json_encode($result->context))->not->toContain('hunter2'); + }); + + it('preserves the channel output format, unlike the formatter', function (): void { + $monolog = new MonologLogger('testing'); + $handler = new TestHandler; + $handler->setFormatter(new JsonFormatter); + $monolog->pushHandler($handler); + $monolog->pushProcessor(new RedactorProcessor(resolve(Redactor::class))); + + $monolog->info('User bob@example.com signed in', ['password' => 'hunter2']); + + $formatted = $handler->getRecords()[0]->formatted; + + expect(json_decode($formatted, true))->toBeArray() + ->and($formatted)->not->toContain('bob@example.com') + ->and($formatted)->not->toContain('hunter2'); + }); +}); + +describe('RedactorTap', function (): void { + it('adds the processor without replacing the formatter', function (): void { + $monolog = new MonologLogger('testing'); + $handler = new TestHandler; + $handler->setFormatter($json = new JsonFormatter); + $monolog->pushHandler($handler); + + (new RedactorTap)(new Logger($monolog)); + + expect($handler->getFormatter())->toBe($json) + ->and($monolog->getProcessors()[0])->toBeInstanceOf(RedactorProcessor::class); + }); + + it('redacts records logged through the tapped channel', function (): void { + $monolog = new MonologLogger('testing'); + $handler = new TestHandler; + $monolog->pushHandler($handler); + + (new RedactorTap)(new Logger($monolog)); + + $monolog->info('mail bob@example.com', ['password' => 'hunter2']); + + $record = $handler->getRecords()[0]; + + expect($record->message)->toBe('mail [REDACTED]') + ->and($record->context['password'])->toBe('[REDACTED]'); + }); +}); + +describe('RedactorFormatter composition', function (): void { + it('formats every record in a batch', function (): void { + $formatter = new RedactorFormatter; + + $out = $formatter->formatBatch([ + logRecord('one'), + logRecord('two'), + logRecord('three'), + ]); + + expect($out)->toContain('one') + ->and($out)->toContain('two') + ->and($out)->toContain('three') + ->and(substr_count($out, "\n"))->toBe(3); + }); + + it('delegates to an inner formatter when given one', function (): void { + $formatter = new RedactorFormatter(new JsonFormatter); + + $out = $formatter->format(logRecord('mail bob@example.com', ['password' => 'hunter2'])); + + $decoded = json_decode($out, true); + + expect($decoded)->toBeArray() + ->and($decoded['message'])->toBe('mail [REDACTED]') + ->and($decoded['context']['password'])->toBe('[REDACTED]'); + }); + + it('includes extra in its own output', function (): void { + $formatter = new RedactorFormatter; + + $out = $formatter->format(logRecord('hi', [], ['pid' => 42])); + + expect($out)->toContain('"pid":42'); + }); +}); + +describe('RedactorTap on other loggers', function (): void { + it('leaves a logger that is not Monolog alone, since only Monolog takes processors', function (): void { + $psr = new NullLogger; + $logger = new Logger($psr); + + (new RedactorTap)($logger); + + expect($logger->getLogger())->toBe($psr); + }); +}); diff --git a/tests/Feature/RedactorProfileTest.php b/tests/Feature/RedactorProfileTest.php index e57e117..8c9e7c6 100644 --- a/tests/Feature/RedactorProfileTest.php +++ b/tests/Feature/RedactorProfileTest.php @@ -4,19 +4,24 @@ namespace Tests\Feature; +use Kirschbaum\Redactor\Exceptions\ProfileNotFoundException; +use Kirschbaum\Redactor\RedactionContext; use Kirschbaum\Redactor\Redactor; use Kirschbaum\Redactor\RedactorConfig; +use Kirschbaum\Redactor\Strategies\BlockedKeysStrategy; +use Kirschbaum\Redactor\Strategies\Contracts\Strategy; +use Kirschbaum\Redactor\Strategies\SafeKeysStrategy; -describe('Redactor Profile Tests', function () { - beforeEach(function () { +describe('Redactor Profile Tests', function (): void { + beforeEach(function (): void { // Set up the profile-based config structure config()->set('redactor.default_profile', 'default'); config()->set('redactor.profiles', [ 'default' => [ 'enabled' => true, 'strategies' => [ - \Kirschbaum\Redactor\Strategies\SafeKeysStrategy::class, - \Kirschbaum\Redactor\Strategies\BlockedKeysStrategy::class, + SafeKeysStrategy::class, + BlockedKeysStrategy::class, ], 'safe_keys' => ['id', 'uuid'], 'blocked_keys' => ['password', 'secret'], @@ -37,8 +42,8 @@ 'strict' => [ 'enabled' => true, 'strategies' => [ - \Kirschbaum\Redactor\Strategies\SafeKeysStrategy::class, - \Kirschbaum\Redactor\Strategies\BlockedKeysStrategy::class, + SafeKeysStrategy::class, + BlockedKeysStrategy::class, ], 'safe_keys' => ['id'], 'blocked_keys' => ['password', 'secret', 'email', 'name'], @@ -57,8 +62,8 @@ 'performance' => [ 'enabled' => true, 'strategies' => [ - \Kirschbaum\Redactor\Strategies\SafeKeysStrategy::class, - \Kirschbaum\Redactor\Strategies\BlockedKeysStrategy::class, + SafeKeysStrategy::class, + BlockedKeysStrategy::class, // No large object or string strategies for performance ], 'safe_keys' => ['id', 'uuid', 'timestamp', 'level'], @@ -78,7 +83,7 @@ ]); }); - test('it uses the default profile when no profile is specified', function () { + test('it uses the default profile when no profile is specified', function (): void { $redactor = new Redactor; $data = [ @@ -95,7 +100,7 @@ ->and($result['_redacted'])->toBeTrue(); }); - test('it uses the specified profile when provided', function () { + test('it uses the specified profile when provided', function (): void { $redactor = new Redactor; $data = [ @@ -115,7 +120,7 @@ ->and($result['_redacted_keys'])->toContain('name'); }); - test('it respects profile-specific configuration options', function () { + test('it respects profile-specific configuration options', function (): void { $redactor = new Redactor; $data = [ @@ -131,17 +136,17 @@ ->and($result)->not->toHaveKey('_redacted'); // No redaction metadata }); - test('it throws exception for non-existent profile', function () { + test('it throws exception for non-existent profile', function (): void { $redactor = new Redactor; - expect(fn () => $redactor->redact(['test' => 'data'], 'non_existent')) - ->toThrow(\InvalidArgumentException::class, "Redaction profile 'non_existent' not found in configuration."); + expect(fn (): mixed => $redactor->redact(['test' => 'data'], 'non_existent')) + ->toThrow(ProfileNotFoundException::class, 'Redaction profile [non_existent] is not configured.'); }); - test('it can list available profiles', function () { + test('it can list available profiles', function (): void { $redactor = new Redactor; - $profiles = $redactor->getAvailableProfiles(); + $profiles = $redactor->profiles(); expect($profiles)->toBeArray() ->and($profiles)->toContain('default') @@ -150,50 +155,50 @@ ->and($profiles)->toHaveCount(3); }); - test('it can check if a profile exists', function () { + test('it can check if a profile exists', function (): void { $redactor = new Redactor; - expect($redactor->profileExists('default'))->toBeTrue() - ->and($redactor->profileExists('strict'))->toBeTrue() - ->and($redactor->profileExists('performance'))->toBeTrue() - ->and($redactor->profileExists('non_existent'))->toBeFalse(); + expect($redactor->hasProfile('default'))->toBeTrue() + ->and($redactor->hasProfile('strict'))->toBeTrue() + ->and($redactor->hasProfile('performance'))->toBeTrue() + ->and($redactor->hasProfile('non_existent'))->toBeFalse(); }); - test('it loads strategies based on profile configuration', function () { + test('it loads strategies based on profile configuration', function (): void { $redactor = new Redactor; - $defaultStrategies = $redactor->getStrategies('default'); - $performanceStrategies = $redactor->getStrategies('performance'); + $defaultStrategies = $redactor->strategies('default'); + $performanceStrategies = $redactor->strategies('performance'); expect($defaultStrategies)->toBeArray() ->and($performanceStrategies)->toBeArray(); // Both should have safe_keys and blocked_keys - $defaultStrategyNames = array_map(fn ($s) => get_class($s), $defaultStrategies); - $performanceStrategyNames = array_map(fn ($s) => get_class($s), $performanceStrategies); + $defaultStrategyNames = array_map(get_class(...), $defaultStrategies); + $performanceStrategyNames = array_map(get_class(...), $performanceStrategies); - expect($defaultStrategyNames)->toContain('Kirschbaum\Redactor\Strategies\SafeKeysStrategy') - ->and($defaultStrategyNames)->toContain('Kirschbaum\Redactor\Strategies\BlockedKeysStrategy') - ->and($performanceStrategyNames)->toContain('Kirschbaum\Redactor\Strategies\SafeKeysStrategy') - ->and($performanceStrategyNames)->toContain('Kirschbaum\Redactor\Strategies\BlockedKeysStrategy'); + expect($defaultStrategyNames)->toContain(SafeKeysStrategy::class) + ->and($defaultStrategyNames)->toContain(BlockedKeysStrategy::class) + ->and($performanceStrategyNames)->toContain(SafeKeysStrategy::class) + ->and($performanceStrategyNames)->toContain(BlockedKeysStrategy::class); }); - test('it can register custom strategies', function () { + test('it can register custom strategies', function (): void { $redactor = new Redactor; - $customStrategy = new class implements \Kirschbaum\Redactor\Strategies\RedactionStrategyInterface + $customStrategy = new class implements Strategy { public function getPriority(): int { return 10; } - public function shouldHandle(mixed $value, string $key, \Kirschbaum\Redactor\RedactionContext $context): bool + public function shouldHandle(mixed $value, string $key, RedactionContext $context): bool { return $key === 'custom_field'; } - public function handle(mixed $value, string $key, \Kirschbaum\Redactor\RedactionContext $context): mixed + public function handle(mixed $value, string $key, RedactionContext $context): mixed { $context->markRedacted(); @@ -204,15 +209,15 @@ public function handle(mixed $value, string $key, \Kirschbaum\Redactor\Redaction $redactor->registerCustomStrategy('my_custom', $customStrategy); // Test that we can get strategies (should include our custom one for profiles that use it) - $strategies = $redactor->getStrategies(); + $strategies = $redactor->strategies(); expect($strategies)->toBeArray(); }); - test('it handles disabled profile gracefully', function () { + test('it handles disabled profile gracefully', function (): void { // Add a disabled profile config()->set('redactor.profiles.disabled_profile', [ 'enabled' => false, - 'strategies' => [\Kirschbaum\Redactor\Strategies\BlockedKeysStrategy::class], + 'strategies' => [BlockedKeysStrategy::class], 'safe_keys' => [], 'blocked_keys' => ['password'], 'patterns' => [], @@ -235,7 +240,7 @@ public function handle(mixed $value, string $key, \Kirschbaum\Redactor\Redaction expect($result)->toBe($data); }); - test('it validates RedactorConfig creation from profile', function () { + test('it validates RedactorConfig creation from profile', function (): void { $config = RedactorConfig::fromConfig('strict'); expect($config->profile)->toBe('strict') diff --git a/tests/Feature/RedactorRecursionLimitTest.php b/tests/Feature/RedactorRecursionLimitTest.php new file mode 100644 index 0000000..6ddf045 --- /dev/null +++ b/tests/Feature/RedactorRecursionLimitTest.php @@ -0,0 +1,197 @@ + $this, 'password' => 'hunter2']; + } +} + +/** + * Two objects that reference each other rather than themselves. + */ +class PingDto +{ + public ?object $partner = null; + + public function toArray(): array + { + return ['partner' => $this->partner, 'token' => 'abc']; + } +} + +/** + * Returns the same child object twice. This is not a cycle and must not be + * mistaken for one. + */ +class RepeatedChildDto +{ + public function __construct(private readonly object $child) {} + + public function toArray(): array + { + return ['first' => $this->child, 'second' => $this->child]; + } +} + +class LeafDto +{ + public function toArray(): array + { + return ['password' => 'leaf-secret', 'keep' => 'visible']; + } +} + +function recursionProfile(array $overrides = []): array +{ + return array_merge([ + 'enabled' => true, + 'strategies' => [BlockedKeysStrategy::class], + 'safe_keys' => [], + 'blocked_keys' => ['password', '*token*'], + 'patterns' => [], + 'replacement' => '[REDACTED]', + 'mark_redacted' => false, + 'track_redacted_keys' => false, + 'non_redactable_object_behavior' => 'preserve', + 'max_value_length' => null, + 'redact_large_objects' => false, + 'max_object_size' => 100, + 'max_depth' => 32, + 'shannon_entropy' => ['enabled' => false], + ], $overrides); +} + +describe('Recursion limits', function (): void { + beforeEach(function (): void { + config()->set('redactor.profiles.recursion', recursionProfile()); + }); + + it('breaks a self-referencing toArray() instead of exhausting memory', function (): void { + // Before the depth budget and cycle check existed, this exhausted the + // 128 MB memory limit and killed the process with a fatal error. + $result = resolve(Redactor::class)->redact(['dto' => new SelfReferencingDto], 'recursion'); + + expect($result)->toBeArray() + ->and($result['dto'])->toBeArray() + ->and($result['dto']['self'])->toContain('Circular reference') + ->and($result['dto']['self'])->toContain(SelfReferencingDto::class) + ->and($result['dto']['password'])->toBe('[REDACTED]'); + }); + + it('breaks a two-object reference cycle', function (): void { + $a = new PingDto; + $b = new PingDto; + $a->partner = $b; + $b->partner = $a; + + $result = resolve(Redactor::class)->redact(['a' => $a], 'recursion'); + + expect($result['a']['partner']['partner'])->toContain('Circular reference') + ->and($result['a']['token'])->toBe('[REDACTED]'); + }); + + it('still walks the same object twice when it is repeated, not cyclic', function (): void { + $result = resolve(Redactor::class)->redact( + ['parent' => new RepeatedChildDto(new LeafDto)], + 'recursion' + ); + + // Both branches must be fully redacted; neither may be mistaken for a + // cycle just because the same instance appears more than once. + expect($result['parent']['first']['password'])->toBe('[REDACTED]') + ->and($result['parent']['first']['keep'])->toBe('visible') + ->and($result['parent']['second']['password'])->toBe('[REDACTED]') + ->and($result['parent']['second']['keep'])->toBe('visible'); + }); + + it('replaces anything deeper than max_depth', function (): void { + config()->set('redactor.profiles.recursion', recursionProfile(['max_depth' => 4])); + + $payload = ['password' => 'top']; + for ($i = 0; $i < 10; $i++) { + $payload = ['nested' => $payload]; + } + + $result = resolve(Redactor::class)->redact($payload, 'recursion'); + + $json = json_encode($result); + + expect($json)->toContain('Max depth of 4 exceeded') + // The cut-off replaces the subtree, so the deep secret never + // appears in the output at all. + ->and($json)->not->toContain('top'); + }); + + it('leaves payloads shallower than max_depth completely intact', function (): void { + config()->set('redactor.profiles.recursion', recursionProfile(['max_depth' => 6])); + + $result = resolve(Redactor::class)->redact([ + 'a' => ['b' => ['c' => ['d' => ['keep' => 'value', 'password' => 'x']]]], + ], 'recursion'); + + expect($result['a']['b']['c']['d']['keep'])->toBe('value') + ->and($result['a']['b']['c']['d']['password'])->toBe('[REDACTED]') + ->and(json_encode($result))->not->toContain('Max depth'); + }); + + it('survives a deeply nested payload that would previously blow the stack', function (): void { + $payload = 'leaf'; + for ($i = 0; $i < 20_000; $i++) { + $payload = ['n' => $payload]; + } + + $before = memory_get_usage(); + $result = resolve(Redactor::class)->redact($payload, 'recursion'); + $growth = (memory_get_usage() - $before) / 1_048_576; + + expect(json_encode($result))->toContain('Max depth of 32 exceeded') + // The walk stops at 32 levels, so memory does not track input depth. + ->and($growth)->toBeLessThan(16.0); + }); + + it('marks the payload as redacted when the depth limit trips', function (): void { + config()->set('redactor.profiles.recursion', recursionProfile([ + 'max_depth' => 2, + 'mark_redacted' => true, + ])); + + $result = resolve(Redactor::class)->redact( + ['a' => ['b' => ['c' => ['harmless' => 'value']]]], + 'recursion' + ); + + expect($result)->toHaveKey('_redacted') + ->and($result['_redacted'])->toBeTrue(); + }); + + it('defaults max_depth when a profile does not set one', function (): void { + $profile = recursionProfile(); + unset($profile['max_depth']); + config()->set('redactor.profiles.recursion_default', $profile); + + expect(RedactorConfig::fromConfig('recursion_default')->maxDepth) + ->toBe(RedactorConfig::DEFAULT_MAX_DEPTH); + }); + + it('rejects a non-positive max_depth', function (): void { + config()->set('redactor.profiles.recursion_bad', recursionProfile(['max_depth' => 0])); + + expect(fn (): RedactorConfig => RedactorConfig::fromConfig('recursion_bad')) + ->toThrow(\InvalidArgumentException::class, 'profiles.recursion_bad.max_depth'); + }); +}); diff --git a/tests/Feature/RedactorRegionPacksTest.php b/tests/Feature/RedactorRegionPacksTest.php new file mode 100644 index 0000000..bd546d2 --- /dev/null +++ b/tests/Feature/RedactorRegionPacksTest.php @@ -0,0 +1,90 @@ +toBe('NI AB123456C'); + }); + + it('redact every sample of every pack once switched on, and leave every counter-sample alone', function (): void { + $packs = config('redactor.regions'); + config()->set('redactor.profiles.default.regions', array_keys($packs)); + + foreach ($packs as $region => $rules) { + foreach ($rules as $name => $rule) { + foreach ($rule['samples'] as $sample) { + $findings = Redactor::inspect($sample)->findings; + + expect(in_array($name, array_map(fn (MatchFinding $f): string => $f->rule, $findings), true))->toBeTrue("{$region}: {$name} missed {$sample}"); + } + + foreach ($rule['counter_samples'] as $sample) { + $findings = Redactor::inspect($sample)->findings; + + expect(in_array($name, array_map(fn (MatchFinding $f): string => $f->rule, $findings), true))->toBeFalse("{$region}: {$name} matched {$sample}"); + } + } + } + }); + + it('validate cleanly as a whole', function (): void { + config()->set('redactor.profiles.default.regions', array_keys(config('redactor.regions'))); + + expect(resolve(\Kirschbaum\Redactor\Redactor::class)->validateProfiles())->toBe([]); + }); + + it('leave ordinary log text alone with every pack on', function (): void { + config()->set('redactor.profiles.default.regions', array_keys(config('redactor.regions'))); + + foreach ([ + 'started at 1694600000 and 2026-09-14 10:00:00', + 'order 1234567890 ref 987654321', + 'version v10.2.100 build 20260914', + 'request 550e8400-e29b-41d4-a716-446655440000', + ] as $line) { + expect(Redactor::inspect($line)->wasRedacted)->toBeFalse($line); + } + }); + + it('carry the entity through to operators', function (): void { + config()->set('app.key', 'base64:'.base64_encode(random_bytes(32))); + config()->set('redactor.profiles.default.regions', ['gb']); + config()->set('redactor.profiles.default.operators', ['default' => 'redact', 'national_id' => 'hash']); + + expect(Redactor::redact('NI AB123456C'))->toMatch('/^NI \[national_id:[a-z0-9]+\]$/'); + }); + + it('let a profile rule of the same name win', function (): void { + config()->set('redactor.profiles.default.regions', ['gb']); + config()->set('redactor.profiles.default.patterns.uk_vat', '/never-matches-anything-xyz/'); + + expect(Redactor::redact('VAT GB436083107'))->toBe('VAT GB436083107'); + }); + + it('name an unknown region', function (): void { + config()->set('redactor.profiles.default.regions', ['atlantis']); + + expect(fn () => Redactor::redact('x'))->toThrow(ConfigurationException::class, 'atlantis'); + }); + + it('accept a validator registered at runtime', function (): void { + Validator::extend('starts_with_z', fn (string $v): bool => str_starts_with($v, 'Z')); + config()->set('redactor.profiles.default.patterns.zed', ['pattern' => '/\b[A-Z]\d{4}\b/', 'validator' => 'starts_with_z']); + + expect(Redactor::redact('Z1234 and A1234'))->toBe('[REDACTED] and A1234'); + }); + + it('reject an unknown validator name with the known ones listed', function (): void { + config()->set('redactor.profiles.default.patterns.bad', ['pattern' => '/x/', 'validator' => 'mystery']); + + expect(fn () => Redactor::redact('x'))->toThrow(ConfigurationException::class, 'luhn'); + }); +}); diff --git a/tests/Feature/RedactorResultMetadataTest.php b/tests/Feature/RedactorResultMetadataTest.php new file mode 100644 index 0000000..1713875 --- /dev/null +++ b/tests/Feature/RedactorResultMetadataTest.php @@ -0,0 +1,161 @@ + true, + 'strategies' => [BlockedKeysStrategy::class, RegexPatternsStrategy::class], + 'safe_keys' => [], + 'blocked_keys' => ['password'], + 'patterns' => ['email' => '/[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+/'], + 'replacement' => '[REDACTED]', + 'mark_redacted' => true, + 'track_redacted_keys' => true, + 'non_redactable_object_behavior' => 'preserve', + 'max_value_length' => null, + 'redact_large_objects' => false, + 'max_object_size' => 100, + 'shannon_entropy' => ['enabled' => false], + ], $overrides); +} + +describe('Redaction metadata kept out of the payload', function (): void { + beforeEach(function (): void { + config()->set('redactor.profiles.meta', metadataProfile()); + }); + + it('reports what happened without touching the value', function (): void { + $result = resolve(Redactor::class)->inspect( + ['password' => 'hunter2', 'keep' => 'visible'], + 'meta' + ); + + expect($result)->toBeInstanceOf(RedactionResult::class) + ->and($result->wasRedacted)->toBeTrue() + ->and($result->redactedKeys)->toBe(['password']); + }); + + it('reports a clean payload as untouched', function (): void { + $result = resolve(Redactor::class)->inspect(['keep' => 'visible'], 'meta'); + + expect($result->wasRedacted)->toBeFalse() + ->and($result->redactedKeys)->toBe([]) + ->and($result->value)->toBe(['keep' => 'visible']); + }); + + it('carries metadata for a bare string, which markers never could', function (): void { + $result = resolve(Redactor::class)->inspect('mail bob@example.com', 'meta'); + + expect($result->value)->toBe('mail [REDACTED]') + ->and($result->wasRedacted)->toBeTrue(); + }); + + it('reports metadata even when mark_redacted is off', function (): void { + config()->set('redactor.profiles.meta', metadataProfile(['mark_redacted' => false])); + + $result = resolve(Redactor::class)->inspect(['password' => 'x'], 'meta'); + + expect($result->value)->toBe(['password' => '[REDACTED]']) + ->and($result->wasRedacted)->toBeTrue() + ->and($result->redactedKeys)->toBe(['password']); + }); + + it('reports a disabled profile as untouched rather than redacted', function (): void { + config()->set('redactor.profiles.meta', metadataProfile(['enabled' => false])); + + $result = resolve(Redactor::class)->inspect(['password' => 'x'], 'meta'); + + expect($result->wasRedacted)->toBeFalse() + ->and($result->value)->toBe(['password' => 'x']); + }); + + it('keeps redact() returning the bare value', function (): void { + expect(resolve(Redactor::class)->redact(['keep' => 'visible'], 'meta')) + ->toBe(['keep' => 'visible']); + }); +}); + +describe('Legacy markers no longer corrupt the payload', function (): void { + beforeEach(function (): void { + config()->set('redactor.profiles.meta', metadataProfile()); + }); + + it('leaves a list a list', function (): void { + // Adding '_redacted' to a list turns it into a JSON object, breaking + // any consumer with an array schema: + // ["a@b.com","x@y.com"] -> {"0":"...","1":"...","_redacted":true} + $result = resolve(Redactor::class)->redact(['a@b.com', 'x@y.com'], 'meta'); + + expect($result)->toBe(['[REDACTED]', '[REDACTED]']) + ->and(array_is_list($result))->toBeTrue() + ->and(json_encode($result))->toBe('["[REDACTED]","[REDACTED]"]'); + }); + + it('still reports the redaction for a list through the result object', function (): void { + $result = resolve(Redactor::class)->inspect(['a@b.com'], 'meta'); + + expect($result->wasRedacted)->toBeTrue() + ->and($result->value)->toBe(['[REDACTED]']); + }); + + it('does not overwrite a caller key named _redacted', function (): void { + // Previously the caller's value was silently replaced with `true`. + $result = resolve(Redactor::class)->redact([ + 'password' => 'x', + '_redacted' => 'user-data-here', + ], 'meta'); + + expect($result['_redacted'])->toBe('user-data-here') + ->and($result['password'])->toBe('[REDACTED]'); + }); + + it('does not overwrite a caller key named _redacted_keys', function (): void { + $result = resolve(Redactor::class)->redact([ + 'password' => 'x', + '_redacted_keys' => ['mine'], + ], 'meta'); + + expect($result['_redacted_keys'])->toBe(['mine']); + }); + + it('still adds markers to an ordinary associative payload', function (): void { + $result = resolve(Redactor::class)->redact(['password' => 'x', 'keep' => 'y'], 'meta'); + + expect($result['_redacted'])->toBeTrue() + ->and($result['_redacted_keys'])->toBe(['password']) + ->and($result['keep'])->toBe('y'); + }); + + it('adds markers to an empty array so the flag is not lost', function (): void { + // An empty array is technically a list; treating it as one would drop + // the marker for a payload whose contents were removed entirely. + config()->set('redactor.profiles.meta', metadataProfile([ + 'blocked_keys' => [], + 'patterns' => [], + 'strategies' => [BlockedKeysStrategy::class], + ])); + + $result = resolve(Redactor::class)->inspect([], 'meta'); + + expect($result->wasRedacted)->toBeFalse() + ->and($result->value)->toBe([]); + }); + + it('omits _redacted_keys when tracking is disabled', function (): void { + config()->set('redactor.profiles.meta', metadataProfile(['track_redacted_keys' => false])); + + $result = resolve(Redactor::class)->redact(['password' => 'x'], 'meta'); + + expect($result)->toHaveKey('_redacted') + ->and($result)->not->toHaveKey('_redacted_keys'); + }); +}); diff --git a/tests/Feature/RedactorRuleSamplesTest.php b/tests/Feature/RedactorRuleSamplesTest.php new file mode 100644 index 0000000..cc3be9c --- /dev/null +++ b/tests/Feature/RedactorRuleSamplesTest.php @@ -0,0 +1,101 @@ + true, + 'strategies' => [RegexPatternsStrategy::class], + 'safe_keys' => [], + 'blocked_keys' => [], + 'patterns' => $patterns, + 'replacement' => '[REDACTED]', + 'mark_redacted' => false, + 'track_redacted_keys' => false, + 'non_redactable_object_behavior' => 'preserve', + 'max_value_length' => null, + 'redact_large_objects' => false, + 'max_object_size' => 100, + 'shannon_entropy' => ['enabled' => false], + ]; +} + +describe('Rules that carry their own samples', function (): void { + it('passes when every sample is detected and no counter-sample is', function (): void { + config()->set('redactor.profiles.sampled', samplesProfile([ + 'order' => ['pattern' => '/\bORD-\d{6}\b/', 'samples' => ['ref ORD-123456'], 'counter_samples' => ['ORD-12']], + ])); + + expect(resolve(Redactor::class)->validateProfiles())->not->toHaveKey('sampled'); + }); + + it('fails a rule that no longer detects its sample, naming both', function (): void { + config()->set('redactor.profiles.sampled', samplesProfile([ + 'order' => ['pattern' => '/\bORD-\d{6}\b/', 'samples' => ['ref ORD-12']], + ])); + + $errors = resolve(Redactor::class)->validateProfiles(); + + expect($errors['sampled'])->toContain('"order"') + ->and($errors['sampled'])->toContain('does not detect its sample') + ->and($errors['sampled'])->toContain('ORD-12'); + }); + + it('fails a rule that detects a counter-sample', function (): void { + config()->set('redactor.profiles.sampled', samplesProfile([ + 'digits' => ['pattern' => '/\d+/', 'counter_samples' => ['started at 1694600000']], + ])); + + expect(resolve(Redactor::class)->validateProfiles()['sampled'])->toContain('detects its counter-sample'); + }); + + it('checks samples through the real detection path, keywords and validators included', function (): void { + config()->set('redactor.profiles.sampled', samplesProfile([ + 'card' => ['pattern' => '/\b\d{16}\b/', 'validator' => 'luhn', 'samples' => ['1234567890123456']], + 'phone' => ['pattern' => '/\b\d{10}\b/', 'keywords' => ['phone'], 'samples' => ['5558675309']], + ])); + + $errors = resolve(Redactor::class)->validateProfiles()['sampled']; + + expect($errors)->toContain('"card"') + ->and($errors)->toContain('"phone"'); + }); + + it('reports every failing sample, not just the first', function (): void { + config()->set('redactor.profiles.sampled', samplesProfile([ + 'a' => ['pattern' => '/aaa/', 'samples' => ['bbb']], + 'b' => ['pattern' => '/bbb/', 'samples' => ['aaa']], + ])); + + $errors = resolve(Redactor::class)->validateProfiles()['sampled']; + + expect($errors)->toContain('"a"')->and($errors)->toContain('"b"'); + }); + + it('surfaces the failure through redactor:validate', function (): void { + config()->set('redactor.profiles.sampled', samplesProfile([ + 'order' => ['pattern' => '/\bORD-\d{6}\b/', 'samples' => ['nothing here']], + ])); + + $this->artisan('redactor:validate') + ->expectsOutputToContain('does not detect its sample') + ->assertFailed(); + }); + + it('ships every profile with samples that pass', function (): void { + expect(resolve(Redactor::class)->validateProfiles())->toBe([]); + + $rules = RedactorConfig::fromConfig('default')->patterns; + $withSamples = array_filter($rules, fn (PatternRule $r): bool => $r->samples !== []); + + expect(count($withSamples))->toBe(count($rules)); + }); +}); diff --git a/tests/Feature/RedactorRulesetFingerprintTest.php b/tests/Feature/RedactorRulesetFingerprintTest.php new file mode 100644 index 0000000..4d6e755 --- /dev/null +++ b/tests/Feature/RedactorRulesetFingerprintTest.php @@ -0,0 +1,55 @@ + 'file_scan', 'redactor.scan.baseline' => null]); + $this->dir = sys_get_temp_dir().'/redactor_ruleset_'.uniqid(); + mkdir($this->dir); + file_put_contents($this->dir.'/app.env', "KEY=sk_live_4eC39HqLyjWDarjtT1zdp7dc\n"); + }); + + afterEach(fn () => cleanupDirectory($this->dir)); + + it('is stable for the same rules and changes when a rule changes', function (): void { + $before = RedactorConfig::fromConfig('file_scan')->rulesetFingerprint; + + expect($before)->toHaveLength(16) + ->and(RedactorConfig::fromConfig('file_scan')->rulesetFingerprint)->toBe($before); + + config()->set('redactor.profiles.file_scan.patterns.extra', '/x-\d+/'); + + expect(RedactorConfig::fromConfig('file_scan')->rulesetFingerprint)->not->toBe($before); + }); + + it('is carried in JSON and SARIF output', function (): void { + $expected = RedactorConfig::fromConfig('file_scan')->rulesetFingerprint; + + Artisan::call('redactor:scan', ['paths' => [$this->dir], '--output' => 'json']); + expect(json_decode(Artisan::output(), true)[0]['ruleset'])->toBe($expected); + + Artisan::call('redactor:scan', ['paths' => [$this->dir], '--output' => 'sarif']); + expect(json_decode(Artisan::output(), true)['runs'][0]['tool']['driver']['properties']['rulesetFingerprint'])->toBe($expected); + }); + + it('is written into the baseline and a mismatch is warned about', function (): void { + $baseline = $this->dir.'/baseline.json'; + + Artisan::call('redactor:scan', ['paths' => [$this->dir], '--baseline' => $baseline, '--update-baseline' => true]); + + expect(Baseline::load($baseline)->ruleset)->toBe(RedactorConfig::fromConfig('file_scan')->rulesetFingerprint); + + Artisan::call('redactor:scan', ['paths' => [$this->dir], '--baseline' => $baseline]); + expect(Artisan::output())->not->toContain('generated under ruleset'); + + config()->set('redactor.profiles.file_scan.patterns.extra', '/x-\d+/'); + + Artisan::call('redactor:scan', ['paths' => [$this->dir], '--baseline' => $baseline]); + expect(Artisan::output())->toContain('generated under ruleset'); + }); +}); diff --git a/tests/Feature/RedactorSafeKeysTest.php b/tests/Feature/RedactorSafeKeysTest.php new file mode 100644 index 0000000..a4cfc94 --- /dev/null +++ b/tests/Feature/RedactorSafeKeysTest.php @@ -0,0 +1,220 @@ + true, + 'strategies' => [SafeKeysStrategy::class, BlockedKeysStrategy::class, RegexPatternsStrategy::class], + 'safe_keys' => ['trace_id'], + 'blocked_keys' => ['password', '*token*'], + 'patterns' => ['email' => '/[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+/'], + 'replacement' => '[REDACTED]', + 'mark_redacted' => false, + 'track_redacted_keys' => false, + 'non_redactable_object_behavior' => 'preserve', + 'max_value_length' => null, + 'redact_large_objects' => false, + 'max_object_size' => 100, + 'shannon_entropy' => ['enabled' => false], + ], $overrides); +} + +describe('Safe key semantics', function (): void { + it('preserves a scalar under a safe key', function (): void { + config()->set('redactor.profiles.safe', safeKeyProfile()); + + expect(resolve(Redactor::class)->redact(['trace_id' => 'abc-123'], 'safe')) + ->toBe(['trace_id' => 'abc-123']); + }); + + it('preserves the whole subtree under a safe key', function (): void { + config()->set('redactor.profiles.safe', safeKeyProfile(['safe_keys' => ['debug_dump']])); + + // Preservation is now recursive and deliberate. Previously the walk + // descended anyway, because the engine compared value identity to + // decide whether a strategy had handled the value - so "safe" meant + // one thing for a scalar and the opposite for an array. + $result = resolve(Redactor::class)->redact([ + 'debug_dump' => ['password' => 'hunter2', 'nested' => ['api_token' => 'abc']], + ], 'safe'); + + expect($result['debug_dump'])->toBe([ + 'password' => 'hunter2', + 'nested' => ['api_token' => 'abc'], + ]); + }); + + it('still redacts the same keys when they are not under a safe key', function (): void { + config()->set('redactor.profiles.safe', safeKeyProfile(['safe_keys' => ['debug_dump']])); + + expect(resolve(Redactor::class)->redact(['other' => ['password' => 'hunter2']], 'safe')) + ->toBe(['other' => ['password' => '[REDACTED]']]); + }); + + it('matches safe keys case-insensitively', function (): void { + config()->set('redactor.profiles.safe', safeKeyProfile(['safe_keys' => ['trace_id']])); + + expect(resolve(Redactor::class)->redact(['TRACE_ID' => 'abc'], 'safe')) + ->toBe(['TRACE_ID' => 'abc']); + }); + + it('does not treat a whole-array check as a safe key', function (): void { + // redactArray() evaluates the array itself with an empty key. An empty + // key must never match a safe key, or a stray '' entry would preserve + // the entire payload. + config()->set('redactor.profiles.safe', safeKeyProfile(['safe_keys' => ['']])); + + expect(resolve(Redactor::class)->redact(['password' => 'hunter2'], 'safe')) + ->toBe(['password' => '[REDACTED]']); + }); + + it('supports the wildcard patterns the README documents', function (): void { + // The README's Wildcard Patterns section has always said both + // BlockedKeysStrategy and SafeKeysStrategy support them. SafeKeys was + // a strict in_array(), so '*_count' and 'meta_*' were redacted - + // documented behaviour that did not exist. + config()->set('redactor.profiles.safe', safeKeyProfile([ + 'safe_keys' => ['*_count', 'meta_*'], + 'blocked_keys' => ['*count*', 'meta*'], + ])); + + expect(resolve(Redactor::class)->redact([ + 'item_count' => 5, + 'meta_info' => 'x', + 'other_field' => 'y', + ], 'safe'))->toBe([ + 'item_count' => 5, + 'meta_info' => 'x', + 'other_field' => 'y', + ]); + }); + + it('supports every wildcard shape in safe_keys', function (): void { + config()->set('redactor.profiles.safe', safeKeyProfile([ + 'safe_keys' => ['exact_ok', '*contains*', 'prefix_*', '*_suffix', 'multi_*_wild'], + 'blocked_keys' => ['*'], + ])); + + $result = resolve(Redactor::class)->redact([ + 'exact_ok' => 1, + 'a_contains_b' => 2, + 'prefix_thing' => 3, + 'thing_suffix' => 4, + 'multi_any_wild' => 5, + 'blocked_one' => 6, + ], 'safe'); + + expect($result)->toBe([ + 'exact_ok' => 1, + 'a_contains_b' => 2, + 'prefix_thing' => 3, + 'thing_suffix' => 4, + 'multi_any_wild' => 5, + 'blocked_one' => '[REDACTED]', + ]); + }); + + it('matches safe-key wildcards case-insensitively', function (): void { + config()->set('redactor.profiles.safe', safeKeyProfile([ + 'safe_keys' => ['*_COUNT'], + 'blocked_keys' => ['*'], + ])); + + expect(resolve(Redactor::class)->redact(['item_count' => 5], 'safe')) + ->toBe(['item_count' => 5]); + }); + + it('preserves the subtree under a wildcard-matched safe key', function (): void { + config()->set('redactor.profiles.safe', safeKeyProfile([ + 'safe_keys' => ['debug_*'], + ])); + + expect(resolve(Redactor::class)->redact([ + 'debug_dump' => ['password' => 'hunter2'], + ], 'safe'))->toBe(['debug_dump' => ['password' => 'hunter2']]); + }); + + it('declares SafeKeysStrategy as preserving', function (): void { + expect(new SafeKeysStrategy)->toBeInstanceOf(PreservingStrategy::class); + }); +}); + +describe('Shipped default profile safe keys', function (): void { + it('no longer waves free-text and PII fields through', function (): void { + $safe = RedactorConfig::fromConfig('default')->safeKeys; + + // Each of these used to be safe, so the value was emitted verbatim no + // matter what it contained. + expect($safe)->not->toContain('message') + ->and($safe)->not->toContain('title') + ->and($safe)->not->toContain('url') + ->and($safe)->not->toContain('path') + ->and($safe)->not->toContain('ip') + ->and($safe)->not->toContain('user_agent') + ->and($safe)->not->toContain('source') + ->and($safe)->not->toContain('target'); + }); + + it('actually redacts an email in a message field now', function (): void { + // The headline symptom: with 'message' safe, this address was emitted + // in full while the identical string under any other key was redacted. + $result = resolve(Redactor::class)->redact([ + 'message' => 'User bob@example.com failed to authenticate', + ], 'default'); + + expect($result['message'])->toBe('User [REDACTED] failed to authenticate'); + }); + + it('redacts credentials embedded in a url', function (): void { + $result = resolve(Redactor::class)->redact([ + 'url' => 'https://admin:s3cr3t@internal.example.com/reports', + ], 'default'); + + expect($result['url'])->not->toContain('s3cr3t'); + }); + + it('keeps genuinely structural keys safe', function (): void { + $safe = RedactorConfig::fromConfig('default')->safeKeys; + + expect($safe)->toContain('id') + ->and($safe)->toContain('uuid') + ->and($safe)->toContain('trace_id') + ->and($safe)->toContain('created_at') + ->and($safe)->toContain('level'); + }); + + it('has no key in both safe_keys and blocked_keys in any shipped profile', function (): void { + foreach (RedactorConfig::profiles() as $profile) { + $config = RedactorConfig::fromConfig($profile); + + // session_id was in both lists in the default profile. SafeKeys + // runs first, so the blocked_keys entry was dead configuration. + expect(array_intersect($config->safeKeys, $config->blockedKeys)) + ->toBe([], "profile [{$profile}] lists keys as both safe and blocked"); + } + }); +}); + +describe('redactor:validate catches safe/blocked conflicts', function (): void { + it('fails when a profile lists a key as both safe and blocked', function (): void { + config()->set('redactor.profiles.conflicted', safeKeyProfile([ + 'safe_keys' => ['session_id'], + 'blocked_keys' => ['session_id'], + ])); + + $this->artisan('redactor:validate') + ->expectsOutputToContain('conflicted') + ->assertFailed(); + }); +}); diff --git a/tests/Feature/RedactorSaltTest.php b/tests/Feature/RedactorSaltTest.php new file mode 100644 index 0000000..30bf992 --- /dev/null +++ b/tests/Feature/RedactorSaltTest.php @@ -0,0 +1,41 @@ +set('redactor.pseudonymization.key', testPseudonymizationKey()); + config()->set('redactor.profiles.channel_a', config('redactor.profiles.observability')); + config()->set('redactor.profiles.channel_b', config('redactor.profiles.observability')); + }); + + it('produces the same surrogate for the same value on every profile', function (): void { + $a = resolve(Redactor::class)->redact('alice@customer.com', 'channel_a'); + $b = resolve(Redactor::class)->redact('alice@customer.com', 'channel_b'); + + expect($a)->toMatch('/^u_[a-z0-9]+@customer\.com$/') + ->and($b)->toBe($a); + }); + + it('lets a profile break the correlation with its own salt', function (): void { + config()->set('redactor.profiles.channel_b.pseudonymization', ['salt' => 'export-only']); + + $a = resolve(Redactor::class)->redact('alice@customer.com', 'channel_a'); + $b = resolve(Redactor::class)->redact('alice@customer.com', 'channel_b'); + + expect($b)->toMatch('/^u_[a-z0-9]+@customer\.com$/') + ->and($b)->not->toBe($a); + }); + + it('changes every surrogate when the global salt changes', function (): void { + $before = resolve(Redactor::class)->redact('alice@customer.com', 'channel_a'); + + config()->set('redactor.pseudonymization.salt', 'rotated'); + + expect(resolve(Redactor::class)->redact('alice@customer.com', 'channel_a'))->not->toBe($before); + }); +}); diff --git a/tests/Feature/RedactorScanAllowMarkerTest.php b/tests/Feature/RedactorScanAllowMarkerTest.php new file mode 100644 index 0000000..e5f4616 --- /dev/null +++ b/tests/Feature/RedactorScanAllowMarkerTest.php @@ -0,0 +1,33 @@ +path = tempnam(sys_get_temp_dir(), 'marker'); + }); + + afterEach(fn (): bool => @unlink($this->path)); + + it('drops a finding on a line that carries the marker and keeps the others', function (): void { + file_put_contents($this->path, implode("\n", [ + "\$fixture = 'sk_test_4eC39HqLyjWDarjtT1zdp7dc'; // redactor:allow", + "\$real = 'sk_live_4eC39HqLyjWDarjtT1zdp7dc';", + ])."\n"); + + $findings = resolve(Scanner::class)->scanFile($this->path, 'file_scan')->findings; + + expect($findings)->toHaveCount(1) + ->and($findings[0]->line)->toBe(2); + }); + + it('leaves lines without the marker alone', function (): void { + file_put_contents($this->path, "contact: bob@example.com\n"); + + expect(resolve(Scanner::class)->scanFile($this->path, 'file_scan')->findings)->toHaveCount(1); + }); +}); diff --git a/tests/Feature/RedactorScanCommandTest.php b/tests/Feature/RedactorScanCommandTest.php index 88ab417..f703cc3 100644 --- a/tests/Feature/RedactorScanCommandTest.php +++ b/tests/Feature/RedactorScanCommandTest.php @@ -1,423 +1,515 @@ - 'file_scan']); + config(['redactor.scan.baseline' => null]); + }); - // Create unreadable test files dynamically - $unreadableContent = 'This file should not be readable by the scanner.'; + it('reports a clean file as clean', function (): void { + [$exitCode, $output] = scan(['paths' => [fixturePath('clean-text-file.txt')]]); - $unreadableFile1 = __DIR__.'/fixtures/unreadable-file.txt'; - $unreadableFile2 = __DIR__.'/fixtures/subdirectory/unreadable-file.txt'; + expect($exitCode)->toBe(0) + ->and($output)->toContain('No findings') + ->and($output)->toContain('Files scanned: 1') + ->and($output)->toContain('Total findings: 0'); + }); - file_put_contents($unreadableFile1, $unreadableContent); - file_put_contents($unreadableFile2, $unreadableContent); + it('names the rule and the line for each finding', function (): void { + // The old output was one opaque row per file - "FINDINGS 1 " - + // with no way to know which rule fired or where to look. + [$exitCode, $output] = scan(['paths' => [fixturePath('sensitive-api-keys.txt')]]); - chmod($unreadableFile1, 0000); - chmod($unreadableFile2, 0000); + expect($exitCode)->toBe(0) + ->and($output)->toContain('Rule') + ->and($output)->toContain('Location') + ->and($output)->toMatch('/sensitive-api-keys\.txt:\d+:\d+/'); }); - afterEach(function () { - // Clean up unreadable test files - $unreadableFile1 = __DIR__.'/fixtures/unreadable-file.txt'; - $unreadableFile2 = __DIR__.'/fixtures/subdirectory/unreadable-file.txt'; + it('locates a secret on the line it is actually on', function (): void { + $dir = sys_get_temp_dir().'/redactor_scan_'.uniqid(); + mkdir($dir); + file_put_contents($dir.'/app.env', "APP_NAME=demo\nAPP_ENV=local\nAWS_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE\n"); - if (file_exists($unreadableFile1)) { - chmod($unreadableFile1, 0644); - unlink($unreadableFile1); - } + [, $output] = scan(['paths' => [$dir.'/app.env'], '--output' => 'json']); - if (file_exists($unreadableFile2)) { - chmod($unreadableFile2, 0644); - unlink($unreadableFile2); - } + $findings = json_decode($output, true)[0]['findings']; + + expect($findings)->not->toBeEmpty() + ->and($findings[0]['line'])->toBe(3) + ->and($findings[0]['rule'])->toBe('aws_access_key'); + + cleanupDirectory($dir); }); - it('scans a single clean file and shows clean status', function () { - $exitCode = Artisan::call('redactor:scan', [ - 'paths' => [__DIR__.'/fixtures/clean-text-file.txt'], - ]); + it('shows an excerpt with the secret already redacted', function (): void { + $dir = sys_get_temp_dir().'/redactor_scan_'.uniqid(); + mkdir($dir); + file_put_contents($dir.'/app.env', "AWS_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE\n"); - $output = Artisan::output(); + [, $output] = scan(['paths' => [$dir.'/app.env'], '--output' => 'json']); - expect($exitCode)->toBe(0); - expect($output)->toContain('CLEAN'); - expect($output)->toContain('Files scanned: 1'); - expect($output)->toContain('Files with findings: 0'); + $finding = json_decode($output, true)[0]['findings'][0]; + + expect($finding['excerpt'])->toContain('AWS_ACCESS_KEY_ID') + ->and($finding['excerpt'])->toContain('[REDACTED]') + ->and($finding['excerpt'])->not->toContain('AKIAIOSFODNN7EXAMPLE'); + + cleanupDirectory($dir); + }); + + it('reports several findings in one file separately', function (): void { + [, $output] = scan(['paths' => [fixturePath('personal-info.txt')], '--output' => 'json']); + + $findings = json_decode($output, true)[0]['findings']; + + expect(count($findings))->toBeGreaterThan(1) + ->and(array_unique(array_column($findings, 'rule')))->not->toHaveCount(1); + }); + + it('scans several paths at once', function (): void { + [$exitCode, $output] = scan(['paths' => [ + fixturePath('clean-text-file.txt'), + fixturePath('sensitive-api-keys.txt'), + fixturePath('personal-info.txt'), + ]]); + + expect($exitCode)->toBe(0) + ->and($output)->toContain('Files scanned: 3'); + }); + + it('scans a directory', function (): void { + [$exitCode, $output] = scan(['paths' => [fixturePath('subdirectory')]]); + + expect($exitCode)->toBe(0) + ->and($output)->toContain('Files scanned:'); }); - it('scans a single sensitive file and detects findings', function () { - $exitCode = Artisan::call('redactor:scan', [ - 'paths' => [__DIR__.'/fixtures/sensitive-api-keys.txt'], + it('warns about a path that does not exist', function (): void { + [$exitCode, $output] = scan(['paths' => ['/no/such/path.txt']]); + + expect($exitCode)->toBe(0) + ->and($output)->toContain('Path not found'); + }); + + it('honours --summary-only', function (): void { + [, $output] = scan([ + 'paths' => [fixturePath('sensitive-api-keys.txt')], + '--summary-only' => true, ]); - $output = Artisan::output(); + expect($output)->not->toContain('Location') + ->and($output)->toContain('Total findings:'); + }); - expect($exitCode)->toBe(0); - expect($output)->toContain('FINDINGS'); - expect($output)->toContain('Files scanned: 1'); - expect($output)->toContain('Files with findings: 1'); - }); - - it('scans multiple files with mixed content', function () { - $exitCode = Artisan::call('redactor:scan', [ - 'paths' => [ - __DIR__.'/fixtures/clean-text-file.txt', - __DIR__.'/fixtures/sensitive-api-keys.txt', - __DIR__.'/fixtures/personal-info.txt', - ], + it('honours an explicit --profile', function (): void { + [$exitCode, $output] = scan([ + 'paths' => [fixturePath('clean-text-file.txt')], + '--profile' => 'default', ]); - $output = Artisan::output(); + expect($exitCode)->toBe(0) + ->and($output)->toContain('profile: default'); + }); +}); + +describe('RedactorScanCommand exit codes', function (): void { + beforeEach(function (): void { + config(['redactor.scan.profile' => 'file_scan']); + config(['redactor.scan.baseline' => null]); + }); + + it('exits 0 without --bail even when findings exist', function (): void { + [$exitCode] = scan(['paths' => [fixturePath('sensitive-api-keys.txt')]]); expect($exitCode)->toBe(0); - expect($output)->toContain('CLEAN'); - expect($output)->toContain('FINDINGS'); - expect($output)->toContain('Files scanned: 3'); - expect($output)->toContain('Files with findings: 2'); }); - it('scans a directory and finds all files (excluding filtered files)', function () { - $exitCode = Artisan::call('redactor:scan', [ - 'paths' => [__DIR__.'/fixtures/subdirectory'], + it('exits 1 with --bail when findings exist', function (): void { + [$exitCode] = scan([ + 'paths' => [fixturePath('sensitive-api-keys.txt')], + '--bail' => true, ]); - $output = Artisan::output(); + expect($exitCode)->toBe(1); + }); + + it('exits 0 with --bail when the file is clean', function (): void { + [$exitCode] = scan([ + 'paths' => [fixturePath('clean-text-file.txt')], + '--bail' => true, + ]); expect($exitCode)->toBe(0); - // Should still be 2 files - the large and unreadable files should be filtered out - expect($output)->toContain('Files scanned: 2'); - expect($output)->toContain('nested-secrets.yml'); - expect($output)->toContain('clean-config.yml'); - // Should not contain the filtered files - expect($output)->not->toContain('large-file.txt'); - expect($output)->not->toContain('unreadable-file.txt'); - }); - - it('scans mixed files and directories', function () { - $exitCode = Artisan::call('redactor:scan', [ - 'paths' => [ - __DIR__.'/fixtures/clean-text-file.txt', - __DIR__.'/fixtures/subdirectory', - ], + }); + + it('rejects an unknown output format rather than silently defaulting', function (): void { + [$exitCode, $output] = scan([ + 'paths' => [fixturePath('clean-text-file.txt')], + '--output' => 'yaml', ]); - $output = Artisan::output(); + expect($exitCode)->toBe(1) + ->and($output)->toContain('Unknown --output format'); + }); +}); - expect($exitCode)->toBe(0); - expect($output)->toContain('Files scanned: 3'); - expect($output)->toContain('clean-text-file.txt'); - expect($output)->toContain('nested-secrets.yml'); - expect($output)->toContain('clean-config.yml'); +describe('RedactorScanCommand JSON output', function (): void { + beforeEach(function (): void { + config(['redactor.scan.profile' => 'file_scan']); + config(['redactor.scan.baseline' => null]); }); - it('outputs results in JSON format', function () { - $exitCode = Artisan::call('redactor:scan', [ - 'paths' => [__DIR__.'/fixtures/clean-text-file.txt'], + it('emits parseable JSON with no progress chatter', function (): void { + [, $output] = scan([ + 'paths' => [fixturePath('sensitive-api-keys.txt')], '--output' => 'json', ]); - $output = Artisan::output(); - - expect($exitCode)->toBe(0); + $decoded = json_decode($output, true); - // Extract JSON from output - find the JSON array part - $jsonStart = strpos($output, '['); - $jsonEnd = strrpos($output, ']') + 1; + expect(json_last_error())->toBe(JSON_ERROR_NONE) + ->and($decoded)->toBeArray() + ->and($decoded[0]['status'])->toBe('findings'); + }); - expect($jsonStart)->not->toBeFalse('JSON output should contain an array'); + it('gives every finding a rule, position, excerpt and fingerprint', function (): void { + [, $output] = scan([ + 'paths' => [fixturePath('sensitive-api-keys.txt')], + '--output' => 'json', + ]); - $jsonOutput = substr($output, $jsonStart, $jsonEnd - $jsonStart); - $data = json_decode($jsonOutput, true); + $finding = json_decode($output, true)[0]['findings'][0]; - expect($data)->toBeArray(); - expect($data[0]['status'])->toBe('clean'); - expect($data[0]['findings_count'])->toBe(0); - expect($data[0]['profile'])->toBe('file_scan'); + expect($finding)->toHaveKeys(['rule', 'line', 'column', 'excerpt', 'profile', 'fingerprint']) + ->and($finding['line'])->toBeGreaterThan(0) + ->and($finding['column'])->toBeGreaterThan(0) + ->and($finding['fingerprint'])->toHaveLength(32); }); - it('supports summary-only option', function () { - $exitCode = Artisan::call('redactor:scan', [ - 'paths' => [__DIR__.'/fixtures/clean-text-file.txt'], - '--summary-only' => true, + it('reports a clean file with an empty findings list', function (): void { + [, $output] = scan([ + 'paths' => [fixturePath('clean-text-file.txt')], + '--output' => 'json', ]); - $output = Artisan::output(); + $decoded = json_decode($output, true); - expect($exitCode)->toBe(0); - expect($output)->not->toContain('CLEAN'); // No table shown - expect($output)->toContain('Files scanned: 1'); - expect($output)->toContain('Files with findings: 0'); + expect($decoded[0]['status'])->toBe('clean') + ->and($decoded[0]['findings'])->toBe([]); }); +}); - it('exits with failure code when --bail is used and findings are detected', function () { - $exitCode = Artisan::call('redactor:scan', [ - 'paths' => [__DIR__.'/fixtures/sensitive-api-keys.txt'], - '--bail' => true, +describe('RedactorScanCommand SARIF output', function (): void { + beforeEach(function (): void { + config(['redactor.scan.profile' => 'file_scan']); + config(['redactor.scan.baseline' => null]); + }); + + it('emits a valid SARIF 2.1.0 document', function (): void { + [, $output] = scan([ + 'paths' => [fixturePath('sensitive-api-keys.txt')], + '--output' => 'sarif', ]); - $output = Artisan::output(); + $sarif = json_decode($output, true); - expect($exitCode)->toBe(1); // Failure exit code - expect($output)->toContain('FINDINGS'); - expect($output)->toContain('Files with findings: 1'); + expect(json_last_error())->toBe(JSON_ERROR_NONE) + ->and($sarif['version'])->toBe('2.1.0') + ->and($sarif['runs'][0]['tool']['driver']['name'])->toBe('Redactor') + ->and($sarif['runs'][0]['results'])->not->toBeEmpty(); }); - it('exits with success code when --bail is used and no findings are detected', function () { - $exitCode = Artisan::call('redactor:scan', [ - 'paths' => [__DIR__.'/fixtures/clean-text-file.txt'], - '--bail' => true, + it('locates each result for GitHub code scanning', function (): void { + [, $output] = scan([ + 'paths' => [fixturePath('sensitive-api-keys.txt')], + '--output' => 'sarif', ]); - $output = Artisan::output(); + $result = json_decode($output, true)['runs'][0]['results'][0]; + $region = $result['locations'][0]['physicalLocation']['region']; - expect($exitCode)->toBe(0); // Success exit code - expect($output)->toContain('CLEAN'); - expect($output)->toContain('Files with findings: 0'); + expect($result['ruleId'])->toBeString() + ->and($region['startLine'])->toBeGreaterThan(0) + ->and($region['startColumn'])->toBeGreaterThan(0) + ->and($result['partialFingerprints'])->toHaveKey('redactorFingerprint/v1'); }); - it('uses custom profile when specified', function () { - $exitCode = Artisan::call('redactor:scan', [ - 'paths' => [__DIR__.'/fixtures/clean-text-file.txt'], - '--profile' => 'default', + it('declares every rule it reports', function (): void { + [, $output] = scan([ + 'paths' => [fixturePath('personal-info.txt')], + '--output' => 'sarif', ]); - $output = Artisan::output(); + $run = json_decode($output, true)['runs'][0]; - expect($exitCode)->toBe(0); - expect($output)->toContain('with profile: default'); + $declared = array_column($run['tool']['driver']['rules'], 'id'); + $used = array_unique(array_column($run['results'], 'ruleId')); + + expect(array_diff($used, $declared))->toBe([]); }); - it('defaults to base_path when no paths are provided', function () { - $exitCode = Artisan::call('redactor:scan', []); + it('never puts the secret itself in the report', function (): void { + $dir = sys_get_temp_dir().'/redactor_scan_'.uniqid(); + mkdir($dir); + file_put_contents($dir.'/app.env', "AWS_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE\n"); - $output = Artisan::output(); + [, $output] = scan(['paths' => [$dir.'/app.env'], '--output' => 'sarif']); - expect($exitCode)->toBe(0); - expect($output)->toContain('Scanning paths:'); - expect($output)->toContain('Files scanned:'); + // A SARIF file gets uploaded to GitHub; publishing the secret in it + // would be worse than not scanning at all. + expect($output)->not->toContain('AKIAIOSFODNN7EXAMPLE'); + + cleanupDirectory($dir); }); +}); - it('handles non-existent file gracefully', function () { - $exitCode = Artisan::call('redactor:scan', [ - 'paths' => [ - __DIR__.'/fixtures/non-existent-file.txt', - __DIR__.'/fixtures/clean-text-file.txt', - ], - ]); +describe('RedactorScanCommand baseline', function (): void { + beforeEach(function (): void { + config(['redactor.scan.profile' => 'file_scan']); - $output = Artisan::output(); + $this->baseline = sys_get_temp_dir().'/redactor_baseline_'.uniqid().'.json'; + config(['redactor.scan.baseline' => $this->baseline]); + }); - expect($exitCode)->toBe(0); - expect($output)->toContain('Path not found or not accessible'); - expect($output)->toContain('Files scanned: 1'); // Only the existing file - }); - - it('detects findings in various file types', function () { - $testFiles = [ - 'sensitive-api-keys.txt', - 'personal-info.txt', - 'sensitive-config.json', - 'environment-secrets.env', - 'high-entropy-strings.txt', - 'mixed-content.txt', - 'subdirectory/nested-secrets.yml', - ]; - - foreach ($testFiles as $file) { - $exitCode = Artisan::call('redactor:scan', [ - 'paths' => [__DIR__.'/fixtures/'.$file], - ]); - - $output = Artisan::output(); - - expect($exitCode)->toBe(0); - expect($output)->toContain('FINDINGS'); - expect($output)->toContain('Files with findings: 1'); + afterEach(function (): void { + if (is_file($this->baseline)) { + unlink($this->baseline); } }); - it('identifies clean files correctly', function () { - $testFiles = [ - 'clean-text-file.txt', - 'clean-config.json', - 'subdirectory/clean-config.yml', - ]; + it('writes accepted findings and exits 0', function (): void { + [$exitCode, $output] = scan([ + 'paths' => [fixturePath('sensitive-api-keys.txt')], + '--update-baseline' => true, + ]); - foreach ($testFiles as $file) { - $exitCode = Artisan::call('redactor:scan', [ - 'paths' => [__DIR__.'/fixtures/'.$file], - ]); + expect($exitCode)->toBe(0) + ->and($output)->toContain('Wrote') + ->and(is_file($this->baseline))->toBeTrue(); - $output = Artisan::output(); + $decoded = json_decode((string) file_get_contents($this->baseline), true); - expect($exitCode)->toBe(0); - expect($output)->toContain('CLEAN'); - expect($output)->toContain('Files with findings: 0'); - } + expect($decoded['version'])->toBe(1) + ->and($decoded['findings'])->not->toBeEmpty() + ->and($decoded['findings'][0])->toHaveKeys(['fingerprint', 'rule', 'path']); }); - it('provides detailed findings in JSON output', function () { - $exitCode = Artisan::call('redactor:scan', [ - 'paths' => [__DIR__.'/fixtures/sensitive-api-keys.txt'], - '--output' => 'json', + it('suppresses baselined findings on the next run', function (): void { + scan(['paths' => [fixturePath('sensitive-api-keys.txt')], '--update-baseline' => true]); + + [$exitCode, $output] = scan([ + 'paths' => [fixturePath('sensitive-api-keys.txt')], + '--bail' => true, ]); - $output = Artisan::output(); + // Without a baseline a repo with fixtures or a documented example key + // can never go green, which is how a scanner gets switched off. + expect($exitCode)->toBe(0) + ->and($output)->toContain('Total findings: 0') + ->and($output)->toContain('Suppressed by baseline:'); + }); + + it('still fails on a finding the baseline does not cover', function (): void { + scan(['paths' => [fixturePath('clean-text-file.txt')], '--update-baseline' => true]); - expect($exitCode)->toBe(0); + [$exitCode] = scan([ + 'paths' => [fixturePath('sensitive-api-keys.txt')], + '--bail' => true, + ]); + + expect($exitCode)->toBe(1); + }); - // Extract JSON from output - find the JSON array part - $jsonStart = strpos($output, '['); - $jsonEnd = strrpos($output, ']') + 1; + it('never writes the secret into the baseline file', function (): void { + $dir = sys_get_temp_dir().'/redactor_scan_'.uniqid(); + mkdir($dir); + file_put_contents($dir.'/app.env', "AWS_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE\n"); - expect($jsonStart)->not->toBeFalse('JSON output should contain an array'); + scan(['paths' => [$dir.'/app.env'], '--update-baseline' => true]); - $jsonOutput = substr($output, $jsonStart, $jsonEnd - $jsonStart); - $data = json_decode($jsonOutput, true); + expect(file_get_contents($this->baseline))->not->toContain('AKIAIOSFODNN7EXAMPLE'); - expect($data)->toBeArray(); - expect($data[0]['status'])->toBe('findings'); - expect($data[0]['findings_count'])->toBe(1); - expect($data[0]['findings'][0]['type'])->toBe('full_content_redacted'); - expect($data[0]['profile'])->toBe('file_scan'); + cleanupDirectory($dir); }); - it('scans the original test fixture and finds redactions', function () { - $exitCode = Artisan::call('redactor:scan', [ - 'paths' => [__DIR__.'/fixtures/test-sensitive-file.txt'], - ]); + it('reports a malformed baseline instead of ignoring it', function (): void { + file_put_contents($this->baseline, '{"nope": true}'); + + [$exitCode, $output] = scan(['paths' => [fixturePath('clean-text-file.txt')]]); + + expect($exitCode)->toBe(1) + ->and($output)->toContain('findings'); + }); - $output = Artisan::output(); + it('treats a missing baseline file as empty', function (): void { + [$exitCode] = scan(['paths' => [fixturePath('clean-text-file.txt')]]); expect($exitCode)->toBe(0); - expect($output)->toContain('FINDINGS'); - expect($output)->toContain('Files with findings: 1'); }); - it('truncates long file paths in table output', function () { - // Create a file with a very long path name - $longPath = __DIR__.'/fixtures/this-is-a-very-long-filename-that-should-be-truncated-in-table-output.txt'; - File::copy(__DIR__.'/fixtures/clean-text-file.txt', $longPath); + it('refuses --update-baseline with nowhere to write', function (): void { + config(['redactor.scan.baseline' => null]); - $exitCode = Artisan::call('redactor:scan', [ - 'paths' => [$longPath], + [$exitCode, $output] = scan([ + 'paths' => [fixturePath('clean-text-file.txt')], + '--update-baseline' => true, ]); - $output = Artisan::output(); + expect($exitCode)->toBe(1) + ->and($output)->toContain('--update-baseline needs a path'); + }); +}); - expect($exitCode)->toBe(0); - expect($output)->toContain('...'); +describe('Baseline fingerprints', function (): void { + it('survives the finding moving to a different line', function (): void { + $first = ScanFinding::fingerprint('aws', 'a.env', 'AKIA123'); + $second = ScanFinding::fingerprint('aws', 'a.env', 'AKIA123'); - // Clean up - File::delete($longPath); + expect($first)->toBe($second); }); - it('filters out large and unreadable files during directory scanning', function () { - // Get the count of files when scanning the entire fixtures directory - $exitCode = Artisan::call('redactor:scan', [ - 'paths' => [__DIR__.'/fixtures'], - '--output' => 'json', - ]); + it('differs per rule, per path and per secret', function (): void { + $base = ScanFinding::fingerprint('aws', 'a.env', 'AKIA123'); - $output = Artisan::output(); + expect(ScanFinding::fingerprint('gh', 'a.env', 'AKIA123'))->not->toBe($base) + ->and(ScanFinding::fingerprint('aws', 'b.env', 'AKIA123'))->not->toBe($base) + ->and(ScanFinding::fingerprint('aws', 'a.env', 'AKIA999'))->not->toBe($base); + }); - expect($exitCode)->toBe(0); + it('accepts a plain list of fingerprints as well as objects', function (): void { + $path = sys_get_temp_dir().'/redactor_baseline_'.uniqid().'.json'; + file_put_contents($path, json_encode(['findings' => ['abc123', ['fingerprint' => 'def456']]])); - // Extract JSON from output - $jsonStart = strpos($output, '['); - $jsonEnd = strrpos($output, ']') + 1; + $baseline = Baseline::load($path); - expect($jsonStart)->not->toBeFalse('JSON output should contain an array'); + expect($baseline->fingerprints)->toHaveKeys(['abc123', 'def456']); - $jsonOutput = substr($output, $jsonStart, $jsonEnd - $jsonStart); - $data = json_decode($jsonOutput, true); + unlink($path); + }); +}); - // Verify that large-file.txt and unreadable-file.txt are not in the results - $scannedPaths = collect($data)->pluck('path')->toArray(); +describe('RedactorScanCommand skipped files', function (): void { + beforeEach(function (): void { + config(['redactor.scan.profile' => 'file_scan', 'redactor.scan.baseline' => null]); + + // The collector drops unreadable files before the scanner sees them, so a + // skipped result only reaches the command when a file vanishes in between. + $this->mock(Scanner::class, function (MockInterface $mock): void { + $mock->shouldReceive('scanFile')->andReturnUsing(fn (string $file): ScanResult => str_contains($file, 'clean') + ? new ScanResult($file, [], 'file_scan', skipped: true, error: 'File unreadable') + : new ScanResult($file, [new ScanFinding($file, 'aws_access_key', 1, 1, 'AWS_ACCESS_KEY_ID=[REDACTED]', 'file_scan', 'fp', 'aws_access_key', 0.9)], 'file_scan')); + }); + }); - $foundLargeFile = false; - $foundUnreadableFile = false; + it('warns about a file the scanner could not read, after the findings', function (): void { + [$exitCode, $output] = scan(['paths' => [fixturePath('clean-text-file.txt'), fixturePath('sensitive-api-keys.txt')]]); - foreach ($scannedPaths as $path) { - if (str_contains($path, 'large-file.txt')) { - $foundLargeFile = true; - } - if (str_contains($path, 'unreadable-file.txt')) { - $foundUnreadableFile = true; - } - } + expect($exitCode)->toBe(0) + ->and($output)->toContain('Skipped') + ->and($output)->toContain('File unreadable') + ->and($output)->toContain('aws_access_key'); + }); - // These files should be filtered out due to size/permission constraints - expect($foundLargeFile)->toBeFalse('large-file.txt should be filtered out due to size'); - expect($foundUnreadableFile)->toBeFalse('unreadable-file.txt should be filtered out due to permissions'); + it('marks a skipped file in JUnit output', function (): void { + [, $output] = scan(['paths' => [fixturePath('clean-text-file.txt'), fixturePath('sensitive-api-keys.txt')], '--output' => 'junit']); - // But we should still have scanned other files - expect(count($data))->toBeGreaterThan(0, 'Should have scanned some files'); + expect($output)->toContain('') + ->and($output)->toContain('baseline, 0000); - expect($jsonStart)->not->toBeFalse('JSON output should contain an array'); + expect(fn (): Baseline => Baseline::load($this->baseline)) + ->toThrow(JsonException::class, 'could not be read'); + })->skip(posix_geteuid() === 0, 'chmod does not restrict root'); - $jsonOutput = substr($output, $jsonStart, $jsonEnd - $jsonStart); - $data = json_decode($jsonOutput, true); + it('refuses to write a baseline it cannot encode', function (): void { + $finding = new ScanFinding("bad\xff.env", 'rule', 1, 1, 'x', 'file_scan', 'fp'); - // Should only have the clean file, filtered files should be excluded - expect(count($data))->toBe(1, 'Should only scan the one readable, appropriately-sized file'); - expect($data[0]['path'])->toContain('clean-text-file.txt'); - expect($data[0]['status'])->toBe('clean'); + expect(Baseline::write($this->baseline, [$finding], '2026-01-01T00:00:00+00:00'))->toBeFalse() + ->and(is_file($this->baseline))->toBeFalse(); }); - it('displays skipped status when scanner returns skipped result', function () { - // Mock Scanner to return a skipped result to test the display logic - $mockScanner = Mockery::mock(\Kirschbaum\Redactor\Scanner\Scanner::class); - $mockScanner->shouldReceive('scanFile') - ->once() - ->andReturn(new \Kirschbaum\Redactor\Scanner\ScanResult( - path: 'test-file.txt', - findings: [], - profile: 'test', - skipped: true, - error: 'Test error' - )); + it('fails the command when the baseline could not be written', function (): void { + // A rule name that is not UTF-8 cannot be encoded into the baseline file. + config(['redactor.profiles.file_scan.patterns' => ["k\xffey" => '/demo-secret-\d+/']]); + file_put_contents($this->dir.'/app.env', "x = demo-secret-12345\n"); - $this->app->instance(\Kirschbaum\Redactor\Scanner\Scanner::class, $mockScanner); + [$exitCode, $output] = scan(['paths' => [$this->dir.'/app.env'], '--baseline' => $this->baseline, '--update-baseline' => true]); - $exitCode = Artisan::call('redactor:scan', [ - 'paths' => [__DIR__.'/fixtures/clean-text-file.txt'], - ]); + expect($exitCode)->toBe(1) + ->and($output)->toContain('Could not write baseline file') + ->and(is_file($this->baseline))->toBeFalse(); + }); +}); - $output = Artisan::output(); +describe('Scan results', function (): void { + it('returns itself untouched when there is no baseline or nothing to suppress', function (): void { + $finding = new ScanFinding('a.env', 'rule', 1, 1, 'x', 'file_scan', 'fp'); + $clean = new ScanResult('a.env'); + $dirty = new ScanResult('a.env', [$finding]); - expect($exitCode)->toBe(0); - expect($output)->toContain('SKIPPED'); - expect($output)->toContain('Files scanned: 1'); - expect($output)->toContain('Files with findings: 0'); + expect($clean->withoutBaseline(['fp' => true]))->toBe($clean) + ->and($dirty->withoutBaseline([]))->toBe($dirty) + ->and($dirty->withoutBaseline(['fp' => true])->findings)->toBe([]); + }); + + it('serialises a finding to JSON the same way as toArray', function (): void { + $finding = new ScanFinding('a.env', 'rule', 3, 7, 'x', 'file_scan', 'fp', 'aws_access_key', 0.9, ['pattern matched']); + + expect(json_decode((string) json_encode($finding), true))->toBe($finding->toArray()); }); }); diff --git a/tests/Feature/RedactorScanDecodingTest.php b/tests/Feature/RedactorScanDecodingTest.php new file mode 100644 index 0000000..930fcdf --- /dev/null +++ b/tests/Feature/RedactorScanDecodingTest.php @@ -0,0 +1,70 @@ + 'file_scan', 'redactor.scan.baseline' => null]); + $this->dir = sys_get_temp_dir().'/redactor_decode_'.uniqid(); + mkdir($this->dir); + }); + + afterEach(fn () => cleanupDirectory($this->dir)); + + it('finds a credential URL hidden by JSON escaping', function (): void { + file_put_contents($this->dir.'/config.json', json_encode(['db' => 'postgres://app:s3cr3t@db.internal/app'])); + + $findings = resolve(Scanner::class)->scanFile($this->dir.'/config.json', 'file_scan')->findings; + $rules = array_map(fn (ScanFinding $f): string => $f->rule, $findings); + + expect($rules)->toContain('url_with_auth'); + + $finding = $findings[array_search('url_with_auth', $rules, true)]; + + expect($finding->encoding)->toBe('json') + ->and($finding->line)->toBe(1) + ->and($finding->excerpt)->toStartWith('[json]') + ->and($finding->excerpt)->not->toContain('s3cr3t'); + }); + + it('finds a key inside a base64 value, as in a Kubernetes secret', function (): void { + $encoded = base64_encode('STRIPE_SECRET=sk_live_4eC39HqLyjWDarjtT1zdp7dc'); + file_put_contents($this->dir.'/secret.yml', "apiVersion: v1\nkind: Secret\ndata:\n stripe: {$encoded}\n"); + + $findings = resolve(Scanner::class)->scanFile($this->dir.'/secret.yml', 'file_scan')->findings; + $stripe = array_values(array_filter($findings, fn (ScanFinding $f): bool => $f->rule === 'stripe_key')); + + expect($stripe)->toHaveCount(1) + ->and($stripe[0]->encoding)->toBe('base64') + ->and($stripe[0]->line)->toBe(4) + ->and($stripe[0]->excerpt)->not->toContain('4eC39HqLyjWDarjtT1zdp7dc'); + }); + + it('finds a token hidden by URL encoding', function (): void { + file_put_contents($this->dir.'/access.log', 'GET /cb?next=https%3A%2F%2Fadmin%3Ahunter2%40db.example.com%2Fx HTTP/1.1'."\n"); + + $findings = resolve(Scanner::class)->scanFile($this->dir.'/access.log', 'file_scan')->findings; + + expect(array_map(fn (ScanFinding $f): array => [$f->rule, $f->encoding], $findings))->toContain(['url_with_auth', 'url']); + }); + + it('reports the encoding in JSON output and can be switched off', function (): void { + file_put_contents($this->dir.'/config.json', json_encode(['db' => 'postgres://app:s3cr3t@db.internal/app'])); + + Artisan::call('redactor:scan', ['paths' => [$this->dir], '--output' => 'json']); + $with = json_decode(Artisan::output(), true)[0]['findings']; + + config(['redactor.scan.decode' => false]); + app()->forgetInstance(Scanner::class); + + Artisan::call('redactor:scan', ['paths' => [$this->dir], '--output' => 'json']); + $without = json_decode(Artisan::output(), true)[0]['findings']; + + expect(array_column($with, 'encoding'))->toContain('json') + ->and(array_column($without, 'rule'))->not->toContain('url_with_auth'); + }); +}); diff --git a/tests/Feature/RedactorScanGitTest.php b/tests/Feature/RedactorScanGitTest.php new file mode 100644 index 0000000..3629dcf --- /dev/null +++ b/tests/Feature/RedactorScanGitTest.php @@ -0,0 +1,216 @@ +&1'; + + exec($command, $output, $code); + + if ($code !== 0) { + throw new RuntimeException('git '.implode(' ', $arguments).' failed: '.implode("\n", $output)); + } + + return implode("\n", $output); +} + +function scanGit(string $dir, array $arguments): array +{ + app()->setBasePath($dir); + + $exit = Artisan::call('redactor:scan', $arguments); + + return [$exit, Artisan::output()]; +} + +describe('Git-aware scanning', function (): void { + beforeEach(function (): void { + config(['redactor.scan.profile' => 'file_scan', 'redactor.scan.baseline' => null]); + $this->dir = gitRepo(); + $this->basePath = app()->basePath(); + + file_put_contents($this->dir.'/README.md', "# demo\n"); + git($this->dir, 'add', '.'); + git($this->dir, 'commit', '-q', '-m', 'initial'); + }); + + afterEach(function (): void { + app()->setBasePath($this->basePath); + cleanupDirectory($this->dir); + }); + + it('scans only the lines staged for commit and reports their real line numbers', function (): void { + file_put_contents($this->dir.'/README.md', "# demo\n\nsafe line\nAWS_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE\n"); + file_put_contents($this->dir.'/unstaged.env', "STRIPE=sk_live_4eC39HqLyjWDarjtT1zdp7dc\n"); + git($this->dir, 'add', 'README.md'); + + [$exit, $output] = scanGit($this->dir, ['--staged' => true, '--output' => 'json']); + $results = json_decode($output, true); + + expect($exit)->toBe(0) + ->and($results)->toHaveCount(1) + ->and($results[0]['path'])->toBe('README.md') + ->and($results[0]['findings'])->toHaveCount(1) + ->and($results[0]['findings'][0]['rule'])->toBe('aws_access_key') + ->and($results[0]['findings'][0]['line'])->toBe(4); + }); + + it('fails with --bail on a staged secret, which is what the pre-commit hook relies on', function (): void { + file_put_contents($this->dir.'/.env', "KEY=sk_live_4eC39HqLyjWDarjtT1zdp7dc\n"); + git($this->dir, 'add', '.env'); + + [$exit] = scanGit($this->dir, ['--staged' => true, '--bail' => true, '--output' => 'json']); + + expect($exit)->toBe(1); + }); + + it('ignores a pre-existing secret that the staged change does not touch', function (): void { + file_put_contents($this->dir.'/old.env', "KEY=sk_live_4eC39HqLyjWDarjtT1zdp7dc\n"); + git($this->dir, 'add', 'old.env'); + git($this->dir, 'commit', '-q', '-m', 'oops'); + + file_put_contents($this->dir.'/old.env', "KEY=sk_live_4eC39HqLyjWDarjtT1zdp7dc\nCOMMENT=harmless\n"); + git($this->dir, 'add', 'old.env'); + + [$exit, $output] = scanGit($this->dir, ['--staged' => true, '--bail' => true, '--output' => 'json']); + + expect($exit)->toBe(0) + ->and(json_decode($output, true)[0]['findings'])->toBe([]); + }); + + it('scans what a branch adds over a ref with --diff', function (): void { + git($this->dir, 'checkout', '-q', '-b', 'feature'); + file_put_contents($this->dir.'/config.php', " 'ghp_16C7e42F292c6912E7710c838347Ae178B4a'];\n"); + git($this->dir, 'add', 'config.php'); + git($this->dir, 'commit', '-q', '-m', 'add token'); + + [$exit, $output] = scanGit($this->dir, ['--diff' => 'HEAD~1', '--output' => 'json']); + $results = json_decode($output, true); + + expect($exit)->toBe(0) + ->and($results[0]['path'])->toBe('config.php') + ->and($results[0]['findings'][0]['rule'])->toBe('github_token'); + }); + + it('finds a secret in history even after a later commit removed it', function (): void { + file_put_contents($this->dir.'/.env', "KEY=sk_live_4eC39HqLyjWDarjtT1zdp7dc\n"); + git($this->dir, 'add', '.env'); + git($this->dir, 'commit', '-q', '-m', 'leak'); + $leak = trim(git($this->dir, 'rev-parse', 'HEAD')); + + file_put_contents($this->dir.'/.env', "KEY=rotated\n"); + git($this->dir, 'add', '.env'); + git($this->dir, 'commit', '-q', '-m', 'fix'); + + [, $output] = scanGit($this->dir, ['--history' => '', '--output' => 'json']); + $findings = array_merge(...array_map(fn (array $r) => $r['findings'], json_decode($output, true))); + + expect($findings)->toHaveCount(1) + ->and($findings[0]['rule'])->toBe('stripe_key') + ->and($findings[0]['commit'])->toBe($leak) + ->and($findings[0]['line'])->toBe(1); + }); + + it('accepts a range for --history and a pathspec', function (): void { + file_put_contents($this->dir.'/a.env', "A=sk_live_4eC39HqLyjWDarjtT1zdp7dc\n"); + file_put_contents($this->dir.'/b.env', "B=AKIAIOSFODNN7EXAMPLE\n"); + git($this->dir, 'add', '.'); + git($this->dir, 'commit', '-q', '-m', 'two'); + + [, $output] = scanGit($this->dir, ['--history' => 'HEAD~1..HEAD', 'paths' => ['b.env'], '--output' => 'json']); + $results = json_decode($output, true); + + expect($results)->toHaveCount(1) + ->and($results[0]['path'])->toBe('b.env'); + }); + + it('applies the exclude patterns to git paths too', function (): void { + config(['redactor.scan.exclude_patterns' => ['vendor/*']]); + mkdir($this->dir.'/vendor'); + file_put_contents($this->dir.'/vendor/lib.php', "\$k = 'sk_live_4eC39HqLyjWDarjtT1zdp7dc';\n"); + git($this->dir, 'add', '-f', 'vendor/lib.php'); + + [, $output] = scanGit($this->dir, ['--staged' => true, '--output' => 'json']); + + expect(json_decode($output, true))->toBe([]); + }); + + it('shows the commit in the table location for history scans', function (): void { + file_put_contents($this->dir.'/.env', "KEY=sk_live_4eC39HqLyjWDarjtT1zdp7dc\n"); + git($this->dir, 'add', '.env'); + git($this->dir, 'commit', '-q', '-m', 'leak'); + $short = substr(trim(git($this->dir, 'rev-parse', 'HEAD')), 0, 8); + + [, $output] = scanGit($this->dir, ['--history' => '']); + + expect($output)->toContain("{$short}:.env:1:"); + }); + + it('reports a directory that is not a repository', function (): void { + $plain = sys_get_temp_dir().'/redactor_plain_'.uniqid(); + mkdir($plain); + + try { + [$exit, $output] = scanGit($plain, ['--staged' => true]); + } finally { + cleanupDirectory($plain); + } + + expect($exit)->toBe(1) + ->and($output)->toContain('not inside a git repository'); + }); + + it('emits JUnit XML with a failure per finding', function (): void { + file_put_contents($this->dir.'/.env', "KEY=sk_live_4eC39HqLyjWDarjtT1zdp7dc\nMAIL=bob@example.com\n"); + git($this->dir, 'add', '.env'); + + [, $output] = scanGit($this->dir, ['--staged' => true, '--output' => 'junit']); + + $xml = simplexml_load_string($output); + + expect($xml)->not->toBeFalse() + ->and((string) $xml['failures'])->toBe('2') + ->and((string) $xml->testsuite->testcase['name'])->toBe('.env') + ->and($xml->testsuite->testcase->failure)->toHaveCount(2) + ->and($output)->not->toContain('sk_live_4eC39HqLyjWDarjtT1zdp7dc'); + }); + + it('answers isRepository honestly', function (): void { + expect((new GitRepository($this->dir))->isRepository())->toBeTrue() + ->and((new GitRepository(sys_get_temp_dir()))->isRepository())->toBeFalse(); + }); +}); + +describe('Git failures', function (): void { + it('reports what git said when a ref does not exist', function (): void { + $dir = gitRepo(); + file_put_contents($dir.'/README.md', "hello\n"); + git($dir, 'add', '.'); + git($dir, 'commit', '-q', '-m', 'initial'); + config(['redactor.scan.profile' => 'file_scan', 'redactor.scan.baseline' => null]); + + [$exit, $output] = scanGit($dir, ['--diff' => 'no-such-ref']); + + expect($exit)->toBe(1) + ->and($output)->toContain('git diff failed'); + + cleanupDirectory($dir); + }); +}); diff --git a/tests/Feature/RedactorServiceProviderTest.php b/tests/Feature/RedactorServiceProviderTest.php new file mode 100644 index 0000000..38dced6 --- /dev/null +++ b/tests/Feature/RedactorServiceProviderTest.php @@ -0,0 +1,81 @@ +app['config']->get('redactor.default_profile'); + self::$profileSeenDuringRegister = is_string($profile) ? $profile : null; + + try { + self::$redactorResolvableDuringRegister = $this->app->make(Redactor::class) instanceof Redactor; + } catch (\Throwable) { + self::$redactorResolvableDuringRegister = false; + } + } +} + +describe('RedactorServiceProvider', function (): void { + it('merges package config during register so other providers can read it', function (): void { + ConfigProbeProvider::$profileSeenDuringRegister = null; + + $app = app(); + $app->register(RedactorServiceProvider::class, true); + $app->register(ConfigProbeProvider::class, true); + + expect(ConfigProbeProvider::$profileSeenDuringRegister)->toBe('default'); + }); + + it('binds the redactor early enough to resolve during another register()', function (): void { + ConfigProbeProvider::$redactorResolvableDuringRegister = false; + + $app = app(); + $app->register(RedactorServiceProvider::class, true); + $app->register(ConfigProbeProvider::class, true); + + expect(ConfigProbeProvider::$redactorResolvableDuringRegister)->toBeTrue(); + }); + + it('publishes the config file under the redactor-config tag', function (): void { + $paths = ServiceProvider::pathsToPublish(RedactorServiceProvider::class, 'redactor-config'); + + expect($paths)->not->toBeEmpty() + ->and(array_values($paths)[0])->toEndWith('redactor.php'); + }); + + it('registers the scan command', function (): void { + expect(array_keys(resolve(Kernel::class)->all()))->toContain('redactor:scan') + ->and(resolve(RedactorScanCommand::class))->toBeInstanceOf(RedactorScanCommand::class); + }); +}); + +describe('RedactorServiceProvider without a router', function (): void { + it('registers no middleware alias when nothing has bound a router', function (): void { + $container = new Container; + $provider = new RedactorServiceProvider($container); + + (new \ReflectionMethod($provider, 'registerMiddleware'))->invoke($provider); + + expect($container->bound('router'))->toBeFalse(); + }); +}); diff --git a/tests/Feature/RedactorShannonEntropyTest.php b/tests/Feature/RedactorShannonEntropyTest.php index 42392c8..a9ae288 100644 --- a/tests/Feature/RedactorShannonEntropyTest.php +++ b/tests/Feature/RedactorShannonEntropyTest.php @@ -4,18 +4,20 @@ namespace Tests\Feature; +use Kirschbaum\Redactor\RedactionContext; use Kirschbaum\Redactor\Redactor; use Kirschbaum\Redactor\RedactorConfig; +use Kirschbaum\Redactor\Strategies\SafeKeysStrategy; use Kirschbaum\Redactor\Strategies\ShannonEntropyStrategy; -describe('Shannon Entropy Strategy Tests', function () { - it('redacts high entropy strings like API keys', function () { +describe('Shannon Entropy Strategy Tests', function (): void { + it('redacts high entropy strings like API keys', function (): void { // Explicit profile for high entropy test config()->set('redactor.default_profile', 'entropy_test'); config()->set('redactor.profiles.entropy_test', [ 'enabled' => true, 'strategies' => [ - \Kirschbaum\Redactor\Strategies\ShannonEntropyStrategy::class, + ShannonEntropyStrategy::class, ], 'safe_keys' => [], 'blocked_keys' => [], @@ -51,13 +53,13 @@ ->and($result['_redacted'])->toBeTrue(); }); - it('can be disabled via configuration', function () { + it('can be disabled via configuration', function (): void { // Explicit profile with Shannon entropy disabled config()->set('redactor.default_profile', 'entropy_disabled_test'); config()->set('redactor.profiles.entropy_disabled_test', [ 'enabled' => true, 'strategies' => [ - \Kirschbaum\Redactor\Strategies\ShannonEntropyStrategy::class, + ShannonEntropyStrategy::class, ], 'safe_keys' => [], 'blocked_keys' => [], @@ -89,13 +91,13 @@ ->and($result)->not->toHaveKey('_redacted'); }); - it('respects minimum length threshold', function () { + it('respects minimum length threshold', function (): void { // Explicit profile with higher min_length config()->set('redactor.default_profile', 'min_length_test'); config()->set('redactor.profiles.min_length_test', [ 'enabled' => true, 'strategies' => [ - \Kirschbaum\Redactor\Strategies\ShannonEntropyStrategy::class, + ShannonEntropyStrategy::class, ], 'safe_keys' => [], 'blocked_keys' => [], @@ -129,13 +131,13 @@ ->and($result['_redacted'])->toBeTrue(); }); - it('respects entropy threshold', function () { + it('respects entropy threshold', function (): void { // Explicit profile with very high threshold config()->set('redactor.default_profile', 'high_threshold_test'); config()->set('redactor.profiles.high_threshold_test', [ 'enabled' => true, 'strategies' => [ - \Kirschbaum\Redactor\Strategies\ShannonEntropyStrategy::class, + ShannonEntropyStrategy::class, ], 'safe_keys' => [], 'blocked_keys' => [], @@ -168,13 +170,13 @@ expect($result['medium_entropy'])->toBe('sk-1234567890abcdef1234567890abcdef12345678'); }); - it('allows long hex strings to bypass exclusion patterns', function () { + it('allows long hex strings to bypass exclusion patterns', function (): void { // Explicit profile with hex exclusion pattern config()->set('redactor.default_profile', 'hex_pattern_test'); config()->set('redactor.profiles.hex_pattern_test', [ 'enabled' => true, 'strategies' => [ - \Kirschbaum\Redactor\Strategies\ShannonEntropyStrategy::class, + ShannonEntropyStrategy::class, ], 'safe_keys' => [], 'blocked_keys' => [], @@ -214,14 +216,14 @@ }); }); -describe('Shannon Entropy Common Pattern Detection Tests', function () { - it('skips common patterns despite high entropy', function () { +describe('Shannon Entropy Common Pattern Detection Tests', function (): void { + it('skips common patterns despite high entropy', function (): void { // Explicit profile with default exclusion patterns config()->set('redactor.default_profile', 'common_patterns_test'); config()->set('redactor.profiles.common_patterns_test', [ 'enabled' => true, 'strategies' => [ - \Kirschbaum\Redactor\Strategies\ShannonEntropyStrategy::class, + ShannonEntropyStrategy::class, ], 'safe_keys' => [], 'blocked_keys' => [], @@ -269,13 +271,13 @@ ->and($result)->not->toHaveKey('_redacted'); }); - it('skips short hexadecimal hashes', function () { + it('skips short hexadecimal hashes', function (): void { // Explicit profile for hex testing config()->set('redactor.default_profile', 'hex_test'); config()->set('redactor.profiles.hex_test', [ 'enabled' => true, 'strategies' => [ - \Kirschbaum\Redactor\Strategies\ShannonEntropyStrategy::class, + ShannonEntropyStrategy::class, ], 'safe_keys' => [], 'blocked_keys' => [], @@ -317,13 +319,13 @@ ->and($result['long_hex'])->toBe('[REDACTED]'); // Long hex might be redacted if high entropy }); - it('skips whitespace-only strings', function () { + it('skips whitespace-only strings', function (): void { // Explicit profile for whitespace testing config()->set('redactor.default_profile', 'whitespace_test'); config()->set('redactor.profiles.whitespace_test', [ 'enabled' => true, 'strategies' => [ - \Kirschbaum\Redactor\Strategies\ShannonEntropyStrategy::class, + ShannonEntropyStrategy::class, ], 'safe_keys' => [], 'blocked_keys' => [], @@ -366,13 +368,13 @@ ->and($result)->not->toHaveKey('_redacted'); }); - it('skips IPv4 addresses', function () { + it('skips IPv4 addresses', function (): void { // Explicit profile for IP testing config()->set('redactor.default_profile', 'ip_test'); config()->set('redactor.profiles.ip_test', [ 'enabled' => true, 'strategies' => [ - \Kirschbaum\Redactor\Strategies\ShannonEntropyStrategy::class, + ShannonEntropyStrategy::class, ], 'safe_keys' => [], 'blocked_keys' => [], @@ -415,13 +417,13 @@ ->and($result)->not->toHaveKey('_redacted'); }); - it('skips MAC addresses', function () { + it('skips MAC addresses', function (): void { // Explicit profile for MAC testing config()->set('redactor.default_profile', 'mac_test'); config()->set('redactor.profiles.mac_test', [ 'enabled' => true, 'strategies' => [ - \Kirschbaum\Redactor\Strategies\ShannonEntropyStrategy::class, + ShannonEntropyStrategy::class, ], 'safe_keys' => [], 'blocked_keys' => [], @@ -462,13 +464,13 @@ ->and($result)->not->toHaveKey('_redacted'); }); - it('validates isCommonPattern method directly', function () { + it('validates isCommonPattern method directly', function (): void { // Explicit profile for direct method testing config()->set('redactor.default_profile', 'direct_method_test'); config()->set('redactor.profiles.direct_method_test', [ 'enabled' => true, 'strategies' => [ - \Kirschbaum\Redactor\Strategies\ShannonEntropyStrategy::class, + ShannonEntropyStrategy::class, ], 'safe_keys' => [], 'blocked_keys' => [], @@ -495,22 +497,22 @@ $config = RedactorConfig::fromConfig(); // Test URL pattern - expect($redactor->isCommonPattern('https://example.com', $config))->toBeTrue() - ->and($redactor->isCommonPattern('http://test.org', $config))->toBeTrue() - ->and($redactor->isCommonPattern('ftp://example.com', $config))->toBeFalse(); + expect((new ShannonEntropyStrategy)->isCommonPattern('https://example.com', $config))->toBeTrue() + ->and((new ShannonEntropyStrategy)->isCommonPattern('http://test.org', $config))->toBeTrue() + ->and((new ShannonEntropyStrategy)->isCommonPattern('ftp://example.com', $config))->toBeFalse(); // Test UUID pattern - expect($redactor->isCommonPattern('550e8400-e29b-41d4-a716-446655440000', $config))->toBeTrue() - ->and($redactor->isCommonPattern('not-a-uuid-string', $config))->toBeFalse(); + expect((new ShannonEntropyStrategy)->isCommonPattern('550e8400-e29b-41d4-a716-446655440000', $config))->toBeTrue() + ->and((new ShannonEntropyStrategy)->isCommonPattern('not-a-uuid-string', $config))->toBeFalse(); }); - it('uses custom entropy exclusion patterns from configuration', function () { + it('uses custom entropy exclusion patterns from configuration', function (): void { // Explicit profile with custom exclusion patterns config()->set('redactor.default_profile', 'custom_patterns_test'); config()->set('redactor.profiles.custom_patterns_test', [ 'enabled' => true, 'strategies' => [ - \Kirschbaum\Redactor\Strategies\ShannonEntropyStrategy::class, + ShannonEntropyStrategy::class, ], 'safe_keys' => [], 'blocked_keys' => [], @@ -550,12 +552,12 @@ ->and($result['_redacted'])->toBeTrue(); }); - it('covers specific lines 103 and 108 in ShannonEntropyStrategy', function () { + it('covers specific lines 103 and 108 in ShannonEntropyStrategy', function (): void { // Create strategy instance directly to test specific method calls - $strategy = new \Kirschbaum\Redactor\Strategies\ShannonEntropyStrategy; + $strategy = new ShannonEntropyStrategy; // Test line 103: return false when exclusion_patterns is not an array - $config1 = new \Kirschbaum\Redactor\RedactorConfig( + $config1 = new RedactorConfig( enabled: true, safeKeys: [], blockedKeys: [], @@ -576,19 +578,14 @@ strategies: [], profile: 'test' ); - $context1 = new \Kirschbaum\Redactor\RedactionContext($config1); + $context1 = new RedactionContext($config1); - // Use reflection to call isCommonPattern directly - $reflection = new \ReflectionClass($strategy); - $method = $reflection->getMethod('isCommonPattern'); - $method->setAccessible(true); - - // This should hit line 103: return false (exclusion_patterns not array) - $result1 = $method->invoke($strategy, 'test string', $context1); + // exclusion_patterns is not an array, so nothing can be excluded. + $result1 = $strategy->isCommonPattern('test string', $context1->config); expect($result1)->toBeFalse(); // Test line 108: continue when pattern is not a string - $config2 = new \Kirschbaum\Redactor\RedactorConfig( + $config2 = new RedactorConfig( enabled: true, safeKeys: [], blockedKeys: [], @@ -613,22 +610,22 @@ strategies: [], profile: 'test' ); - $context2 = new \Kirschbaum\Redactor\RedactionContext($config2); + $context2 = new RedactionContext($config2); - // This should hit line 108: continue (non-string patterns skipped) - $result2 = $method->invoke($strategy, 'valid_pattern', $context2); + // Non-string patterns are skipped rather than fatal. + $result2 = $strategy->isCommonPattern('valid_pattern', $context2->config); expect($result2)->toBeTrue(); // Should match the valid pattern after skipping non-strings }); }); -describe('Shannon Entropy Algorithm Tests', function () { - it('calculates entropy correctly', function () { +describe('Shannon Entropy Algorithm Tests', function (): void { + it('calculates entropy correctly', function (): void { // Explicit profile for entropy calculation testing config()->set('redactor.default_profile', 'entropy_calc_test'); config()->set('redactor.profiles.entropy_calc_test', [ 'enabled' => true, 'strategies' => [ - \Kirschbaum\Redactor\Strategies\ShannonEntropyStrategy::class, + ShannonEntropyStrategy::class, ], 'safe_keys' => [], 'blocked_keys' => [], @@ -651,20 +648,20 @@ $redactor = new Redactor; // Test known entropy values - expect($redactor->calculateShannonEntropy('aaaa'))->toBe(0.0) // All same character - ->and($redactor->calculateShannonEntropy('abcd'))->toBeGreaterThan(1.9) // Perfect distribution - ->and($redactor->calculateShannonEntropy('abcd'))->toBeLessThan(2.1) // Perfect distribution - ->and($redactor->calculateShannonEntropy('a'))->toBe(0.0) // Single character - ->and($redactor->calculateShannonEntropy(''))->toBe(0.0); // Empty string + expect((new ShannonEntropyStrategy)->calculateShannonEntropy('aaaa'))->toBe(0.0) // All same character + ->and((new ShannonEntropyStrategy)->calculateShannonEntropy('abcd'))->toBeGreaterThan(1.9) // Perfect distribution + ->and((new ShannonEntropyStrategy)->calculateShannonEntropy('abcd'))->toBeLessThan(2.1) // Perfect distribution + ->and((new ShannonEntropyStrategy)->calculateShannonEntropy('a'))->toBe(0.0) // Single character + ->and((new ShannonEntropyStrategy)->calculateShannonEntropy(''))->toBe(0.0); // Empty string }); - it('handles edge cases in entropy calculation', function () { + it('handles edge cases in entropy calculation', function (): void { // Explicit profile for edge case testing config()->set('redactor.default_profile', 'entropy_edge_test'); config()->set('redactor.profiles.entropy_edge_test', [ 'enabled' => true, 'strategies' => [ - \Kirschbaum\Redactor\Strategies\ShannonEntropyStrategy::class, + ShannonEntropyStrategy::class, ], 'safe_keys' => [], 'blocked_keys' => [], @@ -687,26 +684,27 @@ $redactor = new Redactor; // Edge cases - expect($redactor->calculateShannonEntropy(''))->toBe(0.0) - ->and($redactor->calculateShannonEntropy('a'))->toBe(0.0) - ->and($redactor->calculateShannonEntropy('aa'))->toBe(0.0) - ->and($redactor->calculateShannonEntropy('ab'))->toBeGreaterThan(0.9) - ->and($redactor->calculateShannonEntropy('ab'))->toBeLessThan(1.1); + expect((new ShannonEntropyStrategy)->calculateShannonEntropy(''))->toBe(0.0) + ->and((new ShannonEntropyStrategy)->calculateShannonEntropy('a'))->toBe(0.0) + ->and((new ShannonEntropyStrategy)->calculateShannonEntropy('aa'))->toBe(0.0) + ->and((new ShannonEntropyStrategy)->calculateShannonEntropy('ab'))->toBeGreaterThan(0.9) + ->and((new ShannonEntropyStrategy)->calculateShannonEntropy('ab'))->toBeLessThan(1.1); // Unicode characters - $unicodeEntropy = $redactor->calculateShannonEntropy('αβγδ'); + $unicodeEntropy = (new ShannonEntropyStrategy)->calculateShannonEntropy('αβγδ'); expect($unicodeEntropy)->toBeGreaterThan(1.9) ->and($unicodeEntropy)->toBeLessThan(2.1); }); - it('returns zero entropy when no ShannonEntropyStrategy is found during entropy calculation', function () { - // Explicit profile without Shannon entropy strategy + it('computes entropy independently of which strategies a profile enables', function (): void { + // Entropy is a pure property of the string. It used to be routed through + // Redactor, which silently returned 0.0 when the active profile happened + // not to list ShannonEntropyStrategy - a fallback that reported + // high-entropy secrets as perfectly ordered text. config()->set('redactor.default_profile', 'no_entropy_strategy_test'); config()->set('redactor.profiles.no_entropy_strategy_test', [ 'enabled' => true, - 'strategies' => [ - \Kirschbaum\Redactor\Strategies\SafeKeysStrategy::class, - ], + 'strategies' => [SafeKeysStrategy::class], 'safe_keys' => [], 'blocked_keys' => [], 'patterns' => [], @@ -717,28 +715,18 @@ 'max_value_length' => null, 'redact_large_objects' => false, 'max_object_size' => 100, - 'shannon_entropy' => [ - 'enabled' => false, - 'threshold' => 4.0, - 'min_length' => 25, - 'exclusion_patterns' => [], - ], + 'shannon_entropy' => ['enabled' => false], ]); - $redactor = new Redactor; - - // Should return 0.0 when no ShannonEntropyStrategy is found - expect($redactor->calculateShannonEntropy('high-entropy-string-12345'))->toBe(0.0); + expect((new ShannonEntropyStrategy)->calculateShannonEntropy('high-entropy-string-12345')) + ->toBeGreaterThan(3.0); }); - it('returns false when no ShannonEntropyStrategy is found during pattern checking', function () { - // Explicit profile without Shannon entropy strategy + it('reports exclusion matches independently of the active profile strategies', function (): void { config()->set('redactor.default_profile', 'no_pattern_strategy_test'); config()->set('redactor.profiles.no_pattern_strategy_test', [ 'enabled' => true, - 'strategies' => [ - \Kirschbaum\Redactor\Strategies\SafeKeysStrategy::class, - ], + 'strategies' => [SafeKeysStrategy::class], 'safe_keys' => [], 'blocked_keys' => [], 'patterns' => [], @@ -751,26 +739,23 @@ 'max_object_size' => 100, 'shannon_entropy' => [ 'enabled' => false, - 'threshold' => 4.0, - 'min_length' => 25, - 'exclusion_patterns' => [], + 'exclusion_patterns' => ['/^https?:\\/\\//'], ], ]); - $redactor = new Redactor; $config = RedactorConfig::fromConfig(); - // Should return false when no ShannonEntropyStrategy is found - expect($redactor->isCommonPattern('https://example.com', $config))->toBeFalse(); + expect((new ShannonEntropyStrategy)->isCommonPattern('https://example.com', $config))->toBeTrue() + ->and((new ShannonEntropyStrategy)->isCommonPattern('not-a-url', $config))->toBeFalse(); }); - it('uses entropy caching for performance optimization', function () { + it('uses entropy caching for performance optimization', function (): void { // Explicit profile for caching test config()->set('redactor.default_profile', 'caching_test'); config()->set('redactor.profiles.caching_test', [ 'enabled' => true, 'strategies' => [ - \Kirschbaum\Redactor\Strategies\ShannonEntropyStrategy::class, + ShannonEntropyStrategy::class, ], 'safe_keys' => [], 'blocked_keys' => [], @@ -795,9 +780,9 @@ $testString = 'sk-1234567890abcdef1234567890abcdef12345678'; // Calculate entropy multiple times - should use caching - $entropy1 = $redactor->calculateShannonEntropy($testString); - $entropy2 = $redactor->calculateShannonEntropy($testString); - $entropy3 = $redactor->calculateShannonEntropy($testString); + $entropy1 = (new ShannonEntropyStrategy)->calculateShannonEntropy($testString); + $entropy2 = (new ShannonEntropyStrategy)->calculateShannonEntropy($testString); + $entropy3 = (new ShannonEntropyStrategy)->calculateShannonEntropy($testString); // All calculations should return the same value expect($entropy1)->toBe($entropy2) @@ -805,10 +790,10 @@ ->and($entropy1)->toBeGreaterThan(4.0); // Should be high entropy }); - it('handles cached entropy return path using direct strategy method calls', function () { + it('handles cached entropy return path using direct strategy method calls', function (): void { // Test the cached entropy return path directly - $strategy = new \Kirschbaum\Redactor\Strategies\ShannonEntropyStrategy; - $config = new \Kirschbaum\Redactor\RedactorConfig( + $strategy = new ShannonEntropyStrategy; + $config = new RedactorConfig( enabled: true, safeKeys: [], blockedKeys: [], @@ -829,29 +814,24 @@ strategies: [], profile: 'test' ); - $context = new \Kirschbaum\Redactor\RedactionContext($config); - - // Use reflection to directly call calculateShannonEntropy - $reflection = new \ReflectionClass($strategy); - $method = $reflection->getMethod('calculateShannonEntropy'); - $method->setAccessible(true); + $context = new RedactionContext($config); $testString = 'test string for entropy calculation'; // First call calculates and caches - $entropy1 = $method->invoke($strategy, $testString, $context); + $entropy1 = $strategy->calculateShannonEntropy($testString, $context); - // Second call should hit cached path - $entropy2 = $method->invoke($strategy, $testString, $context); + // Second call should hit the cached path + $entropy2 = $strategy->calculateShannonEntropy($testString, $context); expect($entropy1)->toBe($entropy2); expect($entropy1)->toBeFloat(); }); - it('handles non-array exclusion patterns gracefully', function () { + it('handles non-array exclusion patterns gracefully', function (): void { // Test when exclusion_patterns is not an array - this should hit line 103 - $strategy = new \Kirschbaum\Redactor\Strategies\ShannonEntropyStrategy; - $config = new \Kirschbaum\Redactor\RedactorConfig( + $strategy = new ShannonEntropyStrategy; + $config = new RedactorConfig( enabled: true, safeKeys: [], blockedKeys: [], @@ -872,25 +852,20 @@ strategies: [], profile: 'test' ); - $context = new \Kirschbaum\Redactor\RedactionContext($config); - - // Use reflection to directly call isCommonPattern to hit line 103 - $reflection = new \ReflectionClass($strategy); - $method = $reflection->getMethod('isCommonPattern'); - $method->setAccessible(true); + $context = new RedactionContext($config); - $result = $method->invoke($strategy, 'test string that is long enough to be processed', $context); + $result = $strategy->isCommonPattern('test string that is long enough to be processed', $context->config); // Should return false when exclusion_patterns is not an array expect($result)->toBeFalse(); }); - it('handles non-string patterns in exclusion patterns array', function () { + it('handles non-string patterns in exclusion patterns array', function (): void { // Test skipping non-string patterns in exclusion_patterns config()->set('redactor.profiles.mixed_exclusion_patterns', [ 'enabled' => true, 'strategies' => [ - \Kirschbaum\Redactor\Strategies\ShannonEntropyStrategy::class, + ShannonEntropyStrategy::class, ], 'safe_keys' => [], 'blocked_keys' => [], @@ -922,12 +897,12 @@ expect($result)->toBeArray(); }); - it('handles long hex strings special case in isCommonPattern method', function () { + it('handles long hex strings special case in isCommonPattern method', function (): void { // Test the special case for long hex strings that should still be redacted config()->set('redactor.profiles.hex_special_case', [ 'enabled' => true, 'strategies' => [ - \Kirschbaum\Redactor\Strategies\ShannonEntropyStrategy::class, + ShannonEntropyStrategy::class, ], 'safe_keys' => [], 'blocked_keys' => [], @@ -962,3 +937,35 @@ ->and($result['_redacted'])->toBeTrue(); }); }); + +describe('Shannon per-token judgement', function (): void { + it('never judges a token shorter than min_length by its entropy, even when asked directly', function (): void { + config()->set('redactor.profiles.judge', [ + 'enabled' => true, + 'strategies' => [ShannonEntropyStrategy::class], + 'safe_keys' => [], + 'blocked_keys' => [], + 'patterns' => [], + 'replacement' => '[REDACTED]', + 'mark_redacted' => false, + 'track_redacted_keys' => false, + 'non_redactable_object_behavior' => 'preserve', + 'max_value_length' => null, + 'redact_large_objects' => false, + 'max_object_size' => 100, + 'shannon_entropy' => ['enabled' => true, 'threshold' => 3.0, 'min_length' => 20, 'exclusion_patterns' => []], + ]); + + $strategy = new class extends ShannonEntropyStrategy + { + public function judge(string $token, RedactionContext $context): bool + { + return $this->shouldRedactByEntropy($token, $context); + } + }; + $context = new RedactionContext(RedactorConfig::fromConfig('judge')); + + expect($strategy->judge('Zx7Qm4Kd9Rb2Vn6', $context))->toBeFalse() + ->and($strategy->judge('Zx7Qm4Kd9Rb2Vn6Tp1Ws8Yc3Hf', $context))->toBeTrue(); + }); +}); diff --git a/tests/Feature/RedactorShippedPatternsTest.php b/tests/Feature/RedactorShippedPatternsTest.php new file mode 100644 index 0000000..7388ae0 --- /dev/null +++ b/tests/Feature/RedactorShippedPatternsTest.php @@ -0,0 +1,119 @@ + 'auth failed for token eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c', + 'aws_access_key' => 'using key AKIAIOSFODNN7EXAMPLE for s3', + 'github_token' => 'pushed with ghp_16C7e42F292c6912E7710c838347Ae178B4a', + 'github_fine_grained' => 'github_pat_11ABCDEFG0123456789_abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ0123', + 'stripe_key' => 'charged via sk_live_4eC39HqLyjWDarjtT1zdp7dc', + 'slack_token' => 'posted with xoxb-1234567890-abcdefghijABCDEFGHIJ', + 'openai_key' => 'model call with sk-proj-abcdefghijklmnopqrstuvwxyz0123', + 'anthropic_key' => 'model call with sk-ant-api03-abcdefghijklmnopqrstuvwxyz', + 'google_api_key' => 'maps with AIzaSyA1234567890abcdefghijklmnopqrstuv', + 'sendgrid_key' => 'mail via SG.abcdefghijklmnopqrstuv.abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQ', + 'bearer' => 'Authorization: Bearer 8f14e45fceea167a5a36dedd4bea2543', + 'private_key' => "cert -----BEGIN RSA PRIVATE KEY-----\nMIIEow\n-----END RSA PRIVATE KEY----- end", + 'email' => 'user bob@example.com signed in', + 'unicode_email' => 'user josé@münchen.de signed in', + 'iban_spaced' => 'refund to DE89 3704 0044 0532 0130 00 please', + 'iban_compact' => 'refund to DE89370400440532013000 please', + 'phone_intl' => 'call +44 20 7946 0958 now', + 'phone_us' => 'call +1 (555) 867-5309 now', + 'phone_e164' => 'call +447946095800 now', + 'phone_labelled' => 'Phone: 5558675309', + 'card' => 'paid with 4111111111111111 ok', + 'ssn' => 'ssn 123-45-6789 on file', + ]; +} + +function shippedInnocents(): array +{ + return [ + 'unix_timestamp' => 'job started at 1694600000 and finished at 1694600123', + 'order_number' => 'order 1234567890 for customer 987654321', + 'date_time' => 'at 2026-09-13 10:00:00 the job ran', + 'version' => 'running v10.2.100 on php 8.5.8', + 'money' => 'total 1,234.56 charged', + 'invalid_card' => 'ref 1234567890123456 is not a card', + 'invalid_ssn' => 'code 000-12-3456 is not an ssn', + 'uuid' => 'request 550e8400-e29b-41d4-a716-446655440000 done', + 'prose' => 'the quick brown fox jumps over the lazy dog', + 'path' => 'wrote /var/www/html/storage/logs/laravel.log', + ]; +} + +describe('Shipped profiles catch credentials in free text', function (): void { + foreach (['default', 'strict', 'observability', 'file_scan'] as $profile) { + it("catches every planted secret with the {$profile} profile", function () use ($profile): void { + config()->set('app.key', 'base64:'.base64_encode(random_bytes(32))); + + foreach (shippedSecrets() as $name => $line) { + $result = resolve(Redactor::class)->inspect($line, $profile); + + expect($result->wasRedacted)->toBeTrue("{$profile} missed {$name}: {$line}"); + } + }); + } + + it('keeps the host and path of a credential URL for any scheme', function (): void { + $result = resolve(Redactor::class)->redact('db at postgres://app:s3cr3t@db.internal:5432/app'); + + expect($result)->toBe('db at postgres://app:[REDACTED]@db.internal:5432/app'); + }); + + it('replaces only the token after Bearer', function (): void { + expect(resolve(Redactor::class)->redact('Authorization: Bearer 8f14e45fceea167a5a36dedd4bea2543')) + ->toBe('Authorization: Bearer [REDACTED]'); + }); + + it('prefers the more specific provider rule when two prefixes overlap', function (): void { + $result = resolve(Redactor::class)->inspect('sk-ant-api03-abcdefghijklmnopqrstuvwxyz'); + + expect($result->findings)->toHaveCount(1) + ->and($result->findings[0]->rule)->toBe('anthropic_key'); + }); +}); + +describe('Shipped profiles leave ordinary log text alone', function (): void { + foreach (['default', 'observability', 'performance'] as $profile) { + it("does not touch any innocent line with the {$profile} profile", function () use ($profile): void { + foreach (shippedInnocents() as $name => $line) { + $result = resolve(Redactor::class)->inspect($line, $profile); + + expect($result->wasRedacted)->toBeFalse("{$profile} redacted {$name}: ".json_encode($result->value)); + } + }); + } + + it('does not mistake a card number for a formatted phone number', function (): void { + // Partial masking keeps the length, spaces included: 15 masked, 4 kept. + expect(resolve(Redactor::class)->redact('paid with 4111 1111 1111 1111 ok')) + ->toBe('paid with ***************1111 ok'); + }); + + it('believes a bare ten-digit run only next to a label', function (): void { + expect(resolve(Redactor::class)->redact('Phone: 5558675309'))->toBe('Phone: [REDACTED]') + ->and(resolve(Redactor::class)->redact('id 5558675309'))->toBe('id 5558675309'); + }); +}); + +describe('The performance profile', function (): void { + it('still catches an email and a bare token but is gated on literals', function (): void { + expect(resolve(Redactor::class)->redact('user bob@example.com', 'performance'))->toBe('user [REDACTED]') + ->and(resolve(Redactor::class)->redact(['t' => str_repeat('Ab1', 12)], 'performance'))->toBe(['t' => '[REDACTED]']); + }); +}); diff --git a/tests/Feature/RedactorSingletonTest.php b/tests/Feature/RedactorSingletonTest.php new file mode 100644 index 0000000..91dc86c --- /dev/null +++ b/tests/Feature/RedactorSingletonTest.php @@ -0,0 +1,103 @@ + true, + 'strategies' => [BlockedKeysStrategy::class], + 'safe_keys' => [], + 'blocked_keys' => ['password'], + 'patterns' => [], + 'replacement' => '[REDACTED]', + 'mark_redacted' => false, + 'track_redacted_keys' => false, + 'non_redactable_object_behavior' => 'preserve', + 'max_value_length' => null, + 'redact_large_objects' => false, + 'max_object_size' => 100, + 'shannon_entropy' => ['enabled' => false], + ], $overrides); +} + +describe('Redactor container binding', function (): void { + it('resolves the redactor as a singleton', function (): void { + expect(resolve(Redactor::class))->toBe(resolve(Redactor::class)); + }); + + it('resolves the scanner as a singleton sharing that redactor', function (): void { + expect(resolve(Scanner::class))->toBe(resolve(Scanner::class)); + }); + + it('reuses strategy instances across calls for the same profile', function (): void { + config()->set('redactor.profiles.singleton_a', singletonProfile()); + + $redactor = resolve(Redactor::class); + + $first = $redactor->strategies('singleton_a'); + $second = $redactor->strategies('singleton_a'); + + expect($first)->toHaveCount(1) + ->and($first[0])->toBe($second[0]); + }); + + it('rebuilds strategies when a profile changes its strategy list', function (): void { + config()->set('redactor.profiles.singleton_b', singletonProfile()); + + $redactor = resolve(Redactor::class); + + expect($redactor->strategies('singleton_b'))->toHaveCount(1); + + config()->set('redactor.profiles.singleton_b', singletonProfile([ + 'strategies' => [SafeKeysStrategy::class, BlockedKeysStrategy::class], + ])); + + $rebuilt = $redactor->strategies('singleton_b'); + + expect($rebuilt)->toHaveCount(2) + ->and($rebuilt[0])->toBeInstanceOf(SafeKeysStrategy::class); + }); + + it('picks up custom strategies registered after the redactor was constructed', function (): void { + $redactor = resolve(Redactor::class); + + // Force construction (and, previously, eager custom-strategy loading) + // before the custom strategy is configured. + $redactor->profiles(); + + config()->set('redactor.custom_strategies', [ + 'late_strategy' => LateRegisteredStrategy::class, + ]); + config()->set('redactor.profiles.singleton_c', singletonProfile([ + 'strategies' => ['late_strategy'], + ])); + + expect($redactor->redact(['anything' => 'value'], 'singleton_c')) + ->toBe(['anything' => 'LATE']); + }); +}); + +class LateRegisteredStrategy implements Strategy +{ + public function shouldHandle(mixed $value, string $key, RedactionContext $context): bool + { + return is_string($value); + } + + public function handle(mixed $value, string $key, RedactionContext $context): mixed + { + $context->markRedacted(); + + return 'LATE'; + } +} diff --git a/tests/Feature/RedactorSpanReplacementTest.php b/tests/Feature/RedactorSpanReplacementTest.php new file mode 100644 index 0000000..c43e1ae --- /dev/null +++ b/tests/Feature/RedactorSpanReplacementTest.php @@ -0,0 +1,343 @@ + true, + 'strategies' => [RegexPatternsStrategy::class], + 'safe_keys' => [], + 'blocked_keys' => [], + 'patterns' => $patterns, + 'replacement' => '[REDACTED]', + 'mark_redacted' => false, + 'track_redacted_keys' => false, + 'non_redactable_object_behavior' => 'preserve', + 'max_value_length' => null, + 'redact_large_objects' => false, + 'max_object_size' => 100, + 'shannon_entropy' => ['enabled' => false], + ], $overrides); +} + +const EMAIL = '/[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+/'; + +describe('Span-level replacement', function (): void { + it('replaces only the match and keeps the surrounding text', function (): void { + config()->set('redactor.profiles.span', spanProfile(['email' => EMAIL])); + + // Previously the entire message became "[REDACTED]", which made the + // package unusable in the log pipeline it ships an integration for. + expect(resolve(Redactor::class)->redact(['msg' => 'User bob@example.com placed order 123'], 'span')) + ->toBe(['msg' => 'User [REDACTED] placed order 123']); + }); + + it('replaces a bare string passed straight to redact()', function (): void { + config()->set('redactor.profiles.span', spanProfile(['email' => EMAIL])); + + expect(resolve(Redactor::class)->redact('User bob@example.com placed order 123', 'span')) + ->toBe('User [REDACTED] placed order 123'); + }); + + it('replaces every occurrence, not just the first', function (): void { + config()->set('redactor.profiles.span', spanProfile(['email' => EMAIL])); + + expect(resolve(Redactor::class)->redact('a@x.com cc b@y.com and c@z.com', 'span')) + ->toBe('[REDACTED] cc [REDACTED] and [REDACTED]'); + }); + + it('applies several rules to the same string', function (): void { + config()->set('redactor.profiles.span', spanProfile([ + 'email' => EMAIL, + 'ssn' => '/\b\d{3}-\d{2}-\d{4}\b/', + ])); + + expect(resolve(Redactor::class)->redact('bob@x.com / 123-45-6789 / keep', 'span')) + ->toBe('[REDACTED] / [REDACTED] / keep'); + }); + + it('leaves a clean string completely untouched', function (): void { + config()->set('redactor.profiles.span', spanProfile(['email' => EMAIL])); + + expect(resolve(Redactor::class)->redact('nothing sensitive here at all', 'span')) + ->toBe('nothing sensitive here at all'); + }); + + it('marks the payload redacted only when something matched', function (): void { + config()->set('redactor.profiles.span', spanProfile(['email' => EMAIL], [ + 'mark_redacted' => true, + 'track_redacted_keys' => true, + ])); + + $hit = resolve(Redactor::class)->redact(['msg' => 'ping bob@x.com'], 'span'); + $miss = resolve(Redactor::class)->redact(['msg' => 'ping nobody'], 'span'); + + expect($hit)->toHaveKey('_redacted') + ->and($hit['_redacted_keys'])->toBe(['msg']) + ->and($miss)->not->toHaveKey('_redacted'); + }); +}); + +describe('Single-pass assembly', function (): void { + it('handles many matches in one value', function (): void { + config()->set('redactor.profiles.span', spanProfile(['email' => EMAIL])); + + $value = 'a@x.com then b@y.com then c@z.com then d@w.com then e@v.com'; + + expect(resolve(Redactor::class)->redact($value, 'span')) + ->toBe('[REDACTED] then [REDACTED] then [REDACTED] then [REDACTED] then [REDACTED]'); + }); + + it('keeps every character between matches intact', function (): void { + config()->set('redactor.profiles.span', spanProfile(['digits' => '/\d+/'])); + + expect(resolve(Redactor::class)->redact('a1b22c333d', 'span')) + ->toBe('a[REDACTED]b[REDACTED]c[REDACTED]d'); + }); + + it('handles a match at the very start and the very end', function (): void { + config()->set('redactor.profiles.span', spanProfile(['digits' => '/\d+/'])); + + expect(resolve(Redactor::class)->redact('1middle2', 'span')) + ->toBe('[REDACTED]middle[REDACTED]') + ->and(resolve(Redactor::class)->redact('9', 'span')) + ->toBe('[REDACTED]'); + }); + + it('handles adjacent matches with nothing between them', function (): void { + config()->set('redactor.profiles.span', spanProfile(['pair' => '/\d\d/'])); + + expect(resolve(Redactor::class)->redact('1234', 'span')) + ->toBe('[REDACTED][REDACTED]'); + }); + + it('returns the subject untouched when every match is preserved', function (): void { + config()->set('redactor.profiles.span', spanProfile([ + 'digits' => ['pattern' => '/\d+/', 'entity' => 'digits'], + ], ['operators' => ['digits' => 'preserve']])); + + expect(resolve(Redactor::class)->redact('a1b22c', 'span'))->toBe('a1b22c'); + }); + + it('replaces only the accepted matches when a validator rejects some', function (): void { + config()->set('redactor.profiles.span', spanProfile([ + 'card' => ['pattern' => '/\b\d{16}\b/', 'validator' => 'luhn'], + ])); + + expect(resolve(Redactor::class)->redact('bad 1234567890123456 good 4111111111111111 end', 'span')) + ->toBe('bad 1234567890123456 good [REDACTED] end'); + }); + + it('reports offsets against the original subject, not the rewritten one', function (): void { + config()->set('redactor.profiles.span', spanProfile(['digits' => '/\d+/'], [ + 'mark_redacted' => true, + 'track_redacted_keys' => true, + ])); + + // The replacement is longer than what it replaces, so an offset taken + // from the output would drift on every match after the first. + $result = resolve(Redactor::class)->inspect(['v' => 'a1b2c3'], 'span'); + + expect(array_map(fn (MatchFinding $f): int => $f->offset, $result->findings))->toBe([1, 3, 5]); + }); +}); + +describe('Pattern rule modes', function (): void { + it('masks the match while preserving its length', function (): void { + config()->set('redactor.profiles.span', spanProfile([ + 'email' => ['pattern' => EMAIL, 'mode' => 'mask'], + ])); + + expect(resolve(Redactor::class)->redact('to bob@x.com now', 'span')) + ->toBe('to ********* now'); // bob@x.com is 9 characters + }); + + it('keeps the trailing characters in partial mode', function (): void { + config()->set('redactor.profiles.span', spanProfile([ + 'card' => ['pattern' => '/\b\d{16}\b/', 'mode' => 'partial', 'keep' => 4], + ])); + + expect(resolve(Redactor::class)->redact('card 4111111111111111 ok', 'span')) + ->toBe('card ************1111 ok'); + }); + + it('masks everything when the match is no longer than keep', function (): void { + config()->set('redactor.profiles.span', spanProfile([ + 'pin' => ['pattern' => '/\b\d{4}\b/', 'mode' => 'partial', 'keep' => 4], + ])); + + expect(resolve(Redactor::class)->redact('pin 1234 ok', 'span')) + ->toBe('pin **** ok'); + }); + + it('deletes the match in remove mode', function (): void { + config()->set('redactor.profiles.span', spanProfile([ + 'email' => ['pattern' => EMAIL, 'mode' => 'remove'], + ])); + + expect(resolve(Redactor::class)->redact('to bob@x.com now', 'span')) + ->toBe('to now'); + }); + + it('still supports replacing the whole value in full mode', function (): void { + config()->set('redactor.profiles.span', spanProfile([ + 'email' => ['pattern' => EMAIL, 'mode' => 'full'], + ])); + + expect(resolve(Redactor::class)->redact('to bob@x.com now', 'span')) + ->toBe('[REDACTED]'); + }); + + it('honours a custom mask character', function (): void { + config()->set('redactor.profiles.span', spanProfile([ + 'card' => ['pattern' => '/\b\d{16}\b/', 'mode' => 'partial', 'keep' => 4, 'mask_character' => '#'], + ])); + + expect(resolve(Redactor::class)->redact('4111111111111111', 'span')) + ->toBe('############1111'); + }); + + it('rejects an unknown mode instead of silently replacing', function (): void { + config()->set('redactor.profiles.span', spanProfile([ + 'email' => ['pattern' => EMAIL, 'mode' => 'obliterate'], + ])); + + expect(fn (): RedactorConfig => RedactorConfig::fromConfig('span')) + ->toThrow(\InvalidArgumentException::class, 'patterns.email.mode'); + }); + + it('rejects a rule with no pattern', function (): void { + config()->set('redactor.profiles.span', spanProfile([ + 'email' => ['mode' => 'mask'], + ])); + + expect(fn (): RedactorConfig => RedactorConfig::fromConfig('span')) + ->toThrow(\InvalidArgumentException::class, 'patterns.email'); + }); + + it('still drops an uncompilable pattern rather than failing the profile', function (): void { + config()->set('redactor.profiles.span', spanProfile([ + 'ok' => EMAIL, + 'broken' => '/[unclosed/', + ])); + + expect(array_keys(RedactorConfig::fromConfig('span')->patterns))->toBe(['ok']); + }); + + it('counts characters, not bytes, when masking', function (): void { + config()->set('redactor.profiles.span', spanProfile([ + 'word' => ['pattern' => '/h\p{L}+/u', 'mode' => PatternRule::MODE_MASK], + ])); + + expect(resolve(Redactor::class)->redact('say héllo', 'span'))->toBe('say *****'); + }); +}); + +describe('Entropy redaction inside a larger string', function (): void { + it('replaces only the high-entropy token', function (): void { + config()->set('redactor.profiles.entropy_span', spanProfile([], [ + 'strategies' => [ShannonEntropyStrategy::class], + 'shannon_entropy' => [ + 'enabled' => true, + 'threshold' => 4.0, + 'min_length' => 20, + 'exclusion_patterns' => [], + ], + ])); + + $result = resolve(Redactor::class)->redact( + ['msg' => 'deploy failed using key Zx7Qm4Kd9Rb2Vn6Tp1Ws8Yc3Hf please rotate'], + 'entropy_span' + ); + + expect($result['msg'])->toBe('deploy failed using key [REDACTED] please rotate'); + }); + + it('still replaces the whole value when it is a single token', function (): void { + config()->set('redactor.profiles.entropy_span', spanProfile([], [ + 'strategies' => [ShannonEntropyStrategy::class], + 'shannon_entropy' => [ + 'enabled' => true, + 'threshold' => 4.0, + 'min_length' => 20, + 'exclusion_patterns' => [], + ], + ])); + + expect(resolve(Redactor::class)->redact(['k' => 'Zx7Qm4Kd9Rb2Vn6Tp1Ws8Yc3Hf'], 'entropy_span')) + ->toBe(['k' => '[REDACTED]']); + }); +}); + +describe('Strategy chaining', function (): void { + it('lets entropy inspect what the regex rules left standing', function (): void { + // The email matches first. Before chaining existed, the regex strategy + // ended the chain and the API key next to it survived. + config()->set('redactor.profiles.chained', spanProfile(['email' => EMAIL], [ + 'strategies' => [RegexPatternsStrategy::class, ShannonEntropyStrategy::class], + 'shannon_entropy' => [ + 'enabled' => true, + 'threshold' => 4.0, + 'min_length' => 20, + 'exclusion_patterns' => [], + ], + ])); + + $result = resolve(Redactor::class)->redact( + ['msg' => 'from bob@example.com key Zx7Qm4Kd9Rb2Vn6Tp1Ws8Yc3Hf end'], + 'chained' + ); + + expect($result['msg'])->toBe('from [REDACTED] key [REDACTED] end'); + }); + + it('stops the chain at a strategy that replaces the whole value', function (): void { + config()->set('redactor.profiles.terminal', spanProfile(['email' => EMAIL], [ + 'strategies' => [ + BlockedKeysStrategy::class, + RegexPatternsStrategy::class, + ], + 'blocked_keys' => ['secret_note'], + ])); + + expect(resolve(Redactor::class)->redact(['secret_note' => 'bob@example.com'], 'terminal')) + ->toBe(['secret_note' => '[REDACTED]']); + }); +}); + +describe('Pattern rule definitions at their edges', function (): void { + it('drops an uncompilable pattern given in the long form too', function (): void { + config()->set('redactor.profiles.span', spanProfile([ + 'ok' => EMAIL, + 'broken' => ['pattern' => '/[unclosed/', 'entity' => 'x'], + ])); + + expect(array_keys(RedactorConfig::fromConfig('span')->patterns))->toBe(['ok']); + }); + + it('falls back to medium confidence when the configured confidence is not a number', function (): void { + config()->set('redactor.profiles.span', spanProfile(['email' => ['pattern' => EMAIL, 'confidence' => 'high']])); + + expect(RedactorConfig::fromConfig('span')->patterns['email']->confidence)->toBe(Confidence::MEDIUM); + }); + + it('masks with an asterisk when the mask character is configured empty', function (): void { + config()->set('redactor.profiles.span', spanProfile([ + 'email' => ['pattern' => EMAIL, 'mode' => PatternRule::MODE_MASK, 'mask_character' => ''], + ])); + + expect(resolve(Redactor::class)->redact('mail bob@example.com', 'span'))->toBe('mail ***************'); + }); +}); diff --git a/tests/Feature/RedactorStrategyTest.php b/tests/Feature/RedactorStrategyTest.php index 307b472..7cfcb8e 100644 --- a/tests/Feature/RedactorStrategyTest.php +++ b/tests/Feature/RedactorStrategyTest.php @@ -6,17 +6,23 @@ use Kirschbaum\Redactor\RedactionContext; use Kirschbaum\Redactor\Redactor; -use Kirschbaum\Redactor\Strategies\RedactionStrategyInterface; - -describe('Redactor Strategy Priority Tests', function () { - it('prioritizes safe_keys over blocked_keys', function () { +use Kirschbaum\Redactor\RedactorConfig; +use Kirschbaum\Redactor\Strategies\BlockedKeysStrategy; +use Kirschbaum\Redactor\Strategies\Contracts\Strategy; +use Kirschbaum\Redactor\Strategies\LargeStringStrategy; +use Kirschbaum\Redactor\Strategies\RegexPatternsStrategy; +use Kirschbaum\Redactor\Strategies\SafeKeysStrategy; +use Kirschbaum\Redactor\Strategies\ShannonEntropyStrategy; + +describe('Redactor Strategy Priority Tests', function (): void { + it('prioritizes safe_keys over blocked_keys', function (): void { // Explicit profile for priority testing config()->set('redactor.default_profile', 'priority_test'); config()->set('redactor.profiles.priority_test', [ 'enabled' => true, 'strategies' => [ - \Kirschbaum\Redactor\Strategies\SafeKeysStrategy::class, - \Kirschbaum\Redactor\Strategies\BlockedKeysStrategy::class, + SafeKeysStrategy::class, + BlockedKeysStrategy::class, ], 'safe_keys' => ['id', 'email'], 'blocked_keys' => ['email'], @@ -51,14 +57,14 @@ ->and($result)->not->toHaveKey('_redacted'); }); - it('prioritizes blocked_keys over regex patterns', function () { + it('prioritizes blocked_keys over regex patterns', function (): void { // Explicit profile for blocked keys vs regex priority config()->set('redactor.default_profile', 'blocked_vs_regex_test'); config()->set('redactor.profiles.blocked_vs_regex_test', [ 'enabled' => true, 'strategies' => [ - \Kirschbaum\Redactor\Strategies\BlockedKeysStrategy::class, - \Kirschbaum\Redactor\Strategies\RegexPatternsStrategy::class, + BlockedKeysStrategy::class, + RegexPatternsStrategy::class, ], 'safe_keys' => [], 'blocked_keys' => ['user_email'], @@ -89,21 +95,23 @@ $result = $redactor->redact($context); - // user_email should be redacted due to blocked_keys - // message should be redacted due to regex pattern + // user_email is redacted by blocked_keys, which replaces the whole + // value because the key itself is the signal. + // message is redacted by the regex pattern, which replaces only the + // address and leaves the sentence readable. expect($result['user_email'])->toBe('[REDACTED]') - ->and($result['message'])->toBe('[REDACTED]') + ->and($result['message'])->toBe('Contact me at [REDACTED]') ->and($result['_redacted'])->toBeTrue(); }); - it('prioritizes regex patterns over shannon entropy', function () { + it('prioritizes regex patterns over shannon entropy', function (): void { // Explicit profile for regex vs entropy priority config()->set('redactor.default_profile', 'regex_vs_entropy_test'); config()->set('redactor.profiles.regex_vs_entropy_test', [ 'enabled' => true, 'strategies' => [ - \Kirschbaum\Redactor\Strategies\RegexPatternsStrategy::class, - \Kirschbaum\Redactor\Strategies\ShannonEntropyStrategy::class, + RegexPatternsStrategy::class, + ShannonEntropyStrategy::class, ], 'safe_keys' => [], 'blocked_keys' => [], @@ -140,15 +148,15 @@ }); }); -describe('Redactor Safe Keys Strategy Tests', function () { - it('never redacts safe keys', function () { +describe('Redactor Safe Keys Strategy Tests', function (): void { + it('never redacts safe keys', function (): void { // Explicit profile for safe keys testing config()->set('redactor.default_profile', 'safe_keys_test'); config()->set('redactor.profiles.safe_keys_test', [ 'enabled' => true, 'strategies' => [ - \Kirschbaum\Redactor\Strategies\SafeKeysStrategy::class, - \Kirschbaum\Redactor\Strategies\BlockedKeysStrategy::class, + SafeKeysStrategy::class, + BlockedKeysStrategy::class, ], 'safe_keys' => ['id', 'uuid', 'created_at', 'updated_at'], 'blocked_keys' => ['password', 'secret'], @@ -188,13 +196,13 @@ ->and($result['_redacted'])->toBeTrue(); }); - it('handles safe keys case-insensitively', function () { + it('handles safe keys case-insensitively', function (): void { // Explicit profile for case-insensitive safe keys testing config()->set('redactor.default_profile', 'safe_keys_case_test'); config()->set('redactor.profiles.safe_keys_case_test', [ 'enabled' => true, 'strategies' => [ - \Kirschbaum\Redactor\Strategies\SafeKeysStrategy::class, + SafeKeysStrategy::class, ], 'safe_keys' => ['id', 'uuid', 'created_at', 'updated_at'], 'blocked_keys' => [], @@ -233,14 +241,14 @@ }); }); -describe('Redactor Blocked Keys Strategy Tests', function () { - it('always redacts blocked keys', function () { +describe('Redactor Blocked Keys Strategy Tests', function (): void { + it('always redacts blocked keys', function (): void { // Explicit profile for blocked keys testing config()->set('redactor.default_profile', 'blocked_keys_test'); config()->set('redactor.profiles.blocked_keys_test', [ 'enabled' => true, 'strategies' => [ - \Kirschbaum\Redactor\Strategies\BlockedKeysStrategy::class, + BlockedKeysStrategy::class, ], 'safe_keys' => [], 'blocked_keys' => ['email', 'ssn', 'ein'], @@ -278,13 +286,13 @@ ->and($result['_redacted'])->toBeTrue(); }); - it('handles blocked keys case-insensitively', function () { + it('handles blocked keys case-insensitively', function (): void { // Explicit profile for case-insensitive blocked keys testing config()->set('redactor.default_profile', 'blocked_keys_case_test'); config()->set('redactor.profiles.blocked_keys_case_test', [ 'enabled' => true, 'strategies' => [ - \Kirschbaum\Redactor\Strategies\BlockedKeysStrategy::class, + BlockedKeysStrategy::class, ], 'safe_keys' => [], 'blocked_keys' => ['email', 'ssn', 'ein'], @@ -321,14 +329,14 @@ }); }); -describe('Redactor Regex Patterns Strategy Tests', function () { - it('redacts strings matching regex patterns', function () { +describe('Redactor Regex Patterns Strategy Tests', function (): void { + it('redacts strings matching regex patterns', function (): void { // Explicit profile for regex patterns testing config()->set('redactor.default_profile', 'regex_patterns_test'); config()->set('redactor.profiles.regex_patterns_test', [ 'enabled' => true, 'strategies' => [ - \Kirschbaum\Redactor\Strategies\RegexPatternsStrategy::class, + RegexPatternsStrategy::class, ], 'safe_keys' => [], 'blocked_keys' => [], @@ -363,20 +371,20 @@ $result = $redactor->redact($context); - expect($result['user_message'])->toBe('[REDACTED]') - ->and($result['payment_info'])->toBe('[REDACTED]') - ->and($result['contact'])->toBe('[REDACTED]') + expect($result['user_message'])->toBe('Contact me at [REDACTED]') + ->and($result['payment_info'])->toBe('Credit card: [REDACTED]') + ->and($result['contact'])->toBe('Call me at [REDACTED]') ->and($result['normal_text'])->toBe('This is just normal text') ->and($result['_redacted'])->toBeTrue(); }); - it('handles multiple patterns in same string', function () { + it('handles multiple patterns in same string', function (): void { // Explicit profile for multiple patterns testing config()->set('redactor.default_profile', 'multiple_patterns_test'); config()->set('redactor.profiles.multiple_patterns_test', [ 'enabled' => true, 'strategies' => [ - \Kirschbaum\Redactor\Strategies\RegexPatternsStrategy::class, + RegexPatternsStrategy::class, ], 'safe_keys' => [], 'blocked_keys' => [], @@ -408,23 +416,24 @@ $result = $redactor->redact($context); - expect($result['contact_info'])->toBe('[REDACTED]') // Contains both email and phone + // Both matches are replaced in place; the labels around them survive. + expect($result['contact_info'])->toBe('Email: [REDACTED], Phone: [REDACTED]') ->and($result['simple_text'])->toBe('No sensitive data here') ->and($result['_redacted'])->toBeTrue(); }); }); -describe('Strategy Management Tests', function () { - it('returns all registered strategies via getStrategies method', function () { +describe('Strategy Management Tests', function (): void { + it('returns all registered strategies via getStrategies method', function (): void { // Explicit profile for strategy management testing config()->set('redactor.default_profile', 'strategy_management_test'); config()->set('redactor.profiles.strategy_management_test', [ 'enabled' => true, 'strategies' => [ - \Kirschbaum\Redactor\Strategies\SafeKeysStrategy::class, - \Kirschbaum\Redactor\Strategies\BlockedKeysStrategy::class, - \Kirschbaum\Redactor\Strategies\RegexPatternsStrategy::class, - \Kirschbaum\Redactor\Strategies\ShannonEntropyStrategy::class, + SafeKeysStrategy::class, + BlockedKeysStrategy::class, + RegexPatternsStrategy::class, + ShannonEntropyStrategy::class, ], 'safe_keys' => ['id'], 'blocked_keys' => ['password'], @@ -447,27 +456,27 @@ ]); $redactor = new Redactor; - $strategies = $redactor->getStrategies(); + $strategies = $redactor->strategies(); expect($strategies)->toBeArray() ->and(count($strategies))->toBe(4); // Verify strategies are in priority order - expect($strategies[0])->toBeInstanceOf(\Kirschbaum\Redactor\Strategies\SafeKeysStrategy::class) - ->and($strategies[1])->toBeInstanceOf(\Kirschbaum\Redactor\Strategies\BlockedKeysStrategy::class) - ->and($strategies[2])->toBeInstanceOf(\Kirschbaum\Redactor\Strategies\RegexPatternsStrategy::class) - ->and($strategies[3])->toBeInstanceOf(\Kirschbaum\Redactor\Strategies\ShannonEntropyStrategy::class); + expect($strategies[0])->toBeInstanceOf(SafeKeysStrategy::class) + ->and($strategies[1])->toBeInstanceOf(BlockedKeysStrategy::class) + ->and($strategies[2])->toBeInstanceOf(RegexPatternsStrategy::class) + ->and($strategies[3])->toBeInstanceOf(ShannonEntropyStrategy::class); }); - it('demonstrates strategy separation by removing a strategy', function () { + it('demonstrates strategy separation by removing a strategy', function (): void { // Explicit profile without Shannon entropy strategy config()->set('redactor.default_profile', 'strategy_removal_test'); config()->set('redactor.profiles.strategy_removal_test', [ 'enabled' => true, 'strategies' => [ - \Kirschbaum\Redactor\Strategies\SafeKeysStrategy::class, - \Kirschbaum\Redactor\Strategies\BlockedKeysStrategy::class, - \Kirschbaum\Redactor\Strategies\RegexPatternsStrategy::class, + SafeKeysStrategy::class, + BlockedKeysStrategy::class, + RegexPatternsStrategy::class, // Note: shannon_entropy strategy is not included ], 'safe_keys' => ['id'], @@ -503,19 +512,19 @@ expect($result['id'])->toBe(12345) // Safe key ->and($result['password'])->toBe('[REDACTED]') // Blocked key - ->and($result['email_text'])->toBe('[REDACTED]') // Regex pattern + ->and($result['email_text'])->toBe('Contact: [REDACTED]') // Regex pattern, span only ->and($result['high_entropy'])->toBe('sk-1234567890abcdef1234567890abcdef12345678') // Not redacted (no Shannon entropy strategy) ->and($result['_redacted'])->toBeTrue(); }); - it('handles edge case where strategy receives unexpected value type', function () { + it('handles edge case where strategy receives unexpected value type', function (): void { // Explicit profile for edge case testing config()->set('redactor.default_profile', 'edge_case_test'); config()->set('redactor.profiles.edge_case_test', [ 'enabled' => true, 'strategies' => [ - \Kirschbaum\Redactor\Strategies\SafeKeysStrategy::class, - \Kirschbaum\Redactor\Strategies\BlockedKeysStrategy::class, + SafeKeysStrategy::class, + BlockedKeysStrategy::class, ], 'safe_keys' => ['id'], 'blocked_keys' => ['password'], @@ -538,7 +547,7 @@ $redactor = new Redactor; // Create a custom strategy that handles unexpected types - $customStrategy = new class implements RedactionStrategyInterface + $customStrategy = new class implements Strategy { public function shouldHandle(mixed $value, string $key, RedactionContext $context): bool { @@ -559,8 +568,8 @@ public function handle(mixed $value, string $key, RedactionContext $context): mi // Update profile to include the custom strategy first config()->set('redactor.profiles.edge_case_test.strategies', [ 'custom_test', // Custom strategy by name - \Kirschbaum\Redactor\Strategies\SafeKeysStrategy::class, - \Kirschbaum\Redactor\Strategies\BlockedKeysStrategy::class, + SafeKeysStrategy::class, + BlockedKeysStrategy::class, ]); $context = [ @@ -580,14 +589,14 @@ public function handle(mixed $value, string $key, RedactionContext $context): mi }); }); -describe('Strategy Edge Cases and Coverage Tests', function () { - beforeEach(function () { +describe('Strategy Edge Cases and Coverage Tests', function (): void { + beforeEach(function (): void { config()->set('redactor.default_profile', 'default'); config()->set('redactor.profiles.default', [ 'enabled' => true, 'strategies' => [ - \Kirschbaum\Redactor\Strategies\SafeKeysStrategy::class, - \Kirschbaum\Redactor\Strategies\BlockedKeysStrategy::class, + SafeKeysStrategy::class, + BlockedKeysStrategy::class, ], 'safe_keys' => [], 'blocked_keys' => [], @@ -603,86 +612,86 @@ public function handle(mixed $value, string $key, RedactionContext $context): mi ]); }); - it('handles non-string strategy classes in profile configuration', function () { + it('handles non-string strategy classes in profile configuration', function (): void { // Test when strategy class is not a string config()->set('redactor.profiles.default.strategies', [ - \Kirschbaum\Redactor\Strategies\SafeKeysStrategy::class, + SafeKeysStrategy::class, 123, // Non-string strategy - should be skipped null, // Non-string strategy - should be skipped - \Kirschbaum\Redactor\Strategies\BlockedKeysStrategy::class, + BlockedKeysStrategy::class, ]); $redactor = new Redactor; - $strategies = $redactor->getStrategies('default'); + $strategies = $redactor->strategies('default'); // Should have 2 strategies (the 2 valid ones), skipping the non-string entries expect($strategies)->toHaveCount(2); }); - it('handles non-existent strategy classes', function () { + it('handles non-existent strategy classes', function (): void { // Test createStrategyInstance returning null for non-existent class config()->set('redactor.profiles.default.strategies', [ 'NonExistentStrategyClass', // This will return null ]); $redactor = new Redactor; - $strategies = $redactor->getStrategies('default'); + $strategies = $redactor->strategies('default'); // Should have no strategies since the class doesn't exist expect($strategies)->toHaveCount(0); }); - it('handles classes that exist but do not implement RedactionStrategyInterface', function () { - // Test the case where class exists but doesn't implement RedactionStrategyInterface + it('handles classes that exist but do not implement Strategy', function (): void { + // Test the case where class exists but doesn't implement Strategy config()->set('redactor.profiles.default.strategies', [ - \stdClass::class, // Valid class but not a RedactionStrategyInterface + \stdClass::class, // Valid class but not a Strategy ]); $redactor = new Redactor; - $strategies = $redactor->getStrategies('default'); + $strategies = $redactor->strategies('default'); - // Should have no strategies since stdClass doesn't implement RedactionStrategyInterface + // Should have no strategies since stdClass doesn't implement Strategy expect($strategies)->toHaveCount(0); }); - it('handles non-array custom_strategies configuration', function () { + it('handles non-array custom_strategies configuration', function (): void { // Test when custom_strategies config is not an array config()->set('redactor.custom_strategies', 'not_an_array'); $redactor = new Redactor; // Should not throw an error and work normally - expect($redactor->getStrategies('default'))->toBeArray(); + expect($redactor->strategies('default'))->toBeArray(); }); - it('handles invalid custom strategy configurations', function () { + it('handles invalid custom strategy configurations', function (): void { // Test various invalid custom strategy configurations config()->set('redactor.custom_strategies', [ 'valid_strategy' => TestValidCustomStrategy::class, 123 => TestValidCustomStrategy::class, // Non-string name 'invalid_class' => 'NonExistentClass', // Class doesn't exist - 'not_strategy' => \stdClass::class, // Not a RedactionStrategyInterface + 'not_strategy' => \stdClass::class, // Not a Strategy 'invalid_type' => 123, // Not a string class name ]); $redactor = new Redactor; // Should only load the valid strategy - $customStrategies = $redactor->getStrategies('default'); + $customStrategies = $redactor->strategies('default'); expect($customStrategies)->toBeArray(); }); - it('handles LargeStringStrategy with non-string input', function () { + it('handles LargeStringStrategy with non-string input', function (): void { // Test guard clause for non-string values in LargeStringStrategy config()->set('redactor.profiles.default.strategies', [ - \Kirschbaum\Redactor\Strategies\LargeStringStrategy::class, + LargeStringStrategy::class, ]); config()->set('redactor.profiles.default.max_value_length', 10); // Use reflection to manually test the strategy with non-string input - $strategy = new \Kirschbaum\Redactor\Strategies\LargeStringStrategy; - $config = \Kirschbaum\Redactor\RedactorConfig::fromConfig('default'); - $context = new \Kirschbaum\Redactor\RedactionContext($config); + $strategy = new LargeStringStrategy; + $config = RedactorConfig::fromConfig('default'); + $context = new RedactionContext($config); // This should trigger the guard clause $result = $strategy->handle(123, 'test_key', $context); @@ -690,38 +699,42 @@ public function handle(mixed $value, string $key, RedactionContext $context): mi expect($result)->toBe(123); // Should return original value }); - it('uses deprecated addStrategy method for backward compatibility', function () { - // Test deprecated addStrategy method + it('registers a named custom strategy and uses it in a profile', function (): void { $redactor = new Redactor; - $customStrategy = new TestValidCustomStrategy; - - $redactor->addStrategy($customStrategy); + $redactor->registerCustomStrategy('valid_custom', new TestValidCustomStrategy); - // Should register the strategy (check that it doesn't throw an error) - $strategies = $redactor->getStrategies('default'); - expect($strategies)->toBeArray(); - }); - - it('uses deprecated removeStrategy method for backward compatibility', function () { - // Test deprecated removeStrategy method - $redactor = new Redactor; + config()->set('redactor.profiles.named_custom', [ + 'enabled' => true, + 'strategies' => ['valid_custom'], + 'safe_keys' => [], + 'blocked_keys' => [], + 'patterns' => [], + 'replacement' => '[REDACTED]', + 'mark_redacted' => false, + 'track_redacted_keys' => false, + 'non_redactable_object_behavior' => 'preserve', + 'max_value_length' => null, + 'redact_large_objects' => false, + 'max_object_size' => 100, + 'shannon_entropy' => ['enabled' => false], + ]); - $redactor->removeStrategy('some_strategy'); + $strategies = $redactor->strategies('named_custom'); - // Should clear cached strategies (no exception should be thrown) - expect($redactor->getStrategies('default'))->toBeArray(); + expect($strategies)->toHaveCount(1) + ->and($strategies[0])->toBeInstanceOf(TestValidCustomStrategy::class); }); }); // Test helper class for strategy tests -class TestValidCustomStrategy implements \Kirschbaum\Redactor\Strategies\RedactionStrategyInterface +class TestValidCustomStrategy implements Strategy { - public function shouldHandle(mixed $value, string $key, \Kirschbaum\Redactor\RedactionContext $context): bool + public function shouldHandle(mixed $value, string $key, RedactionContext $context): bool { return false; } - public function handle(mixed $value, string $key, \Kirschbaum\Redactor\RedactionContext $context): mixed + public function handle(mixed $value, string $key, RedactionContext $context): mixed { return $value; } diff --git a/tests/Feature/RedactorStreamingScanTest.php b/tests/Feature/RedactorStreamingScanTest.php new file mode 100644 index 0000000..be71ce7 --- /dev/null +++ b/tests/Feature/RedactorStreamingScanTest.php @@ -0,0 +1,225 @@ + "line {$i}", range(1, 10)))); + + $reader = new LineWindowReader($path, windowLines: 4, overlapLines: 0); + + $seen = []; + foreach ($reader as [$start, $text]) { + foreach (explode("\n", $text) as $offset => $line) { + $seen[$start + $offset] = $line; + } + } + + expect($seen)->toHaveCount(10) + ->and($seen[1])->toBe('line 1') + ->and($seen[10])->toBe('line 10'); + + cleanupDirectory(dirname($path)); + }); + + it('numbers lines correctly across windows with overlap', function (): void { + $path = scratchFile(implode("\n", array_map(fn (int $i): string => "line {$i}", range(1, 20)))); + + foreach (new LineWindowReader($path, windowLines: 6, overlapLines: 2) as [$start, $text]) { + $first = explode("\n", $text)[0]; + + // The reported start line must actually be the first line of the + // window, or every finding in it is attributed to the wrong place. + expect($first)->toBe("line {$start}"); + } + + cleanupDirectory(dirname($path)); + }); + + it('carries lines forward so a match spanning a boundary survives', function (): void { + $path = scratchFile(implode("\n", array_map(fn (int $i): string => "line {$i}", range(1, 12)))); + + $windows = []; + foreach (new LineWindowReader($path, windowLines: 5, overlapLines: 2) as [$start, $text]) { + $windows[] = [$start, $text]; + } + + expect(count($windows))->toBeGreaterThan(1) + // Window two begins inside window one. + ->and($windows[1][0])->toBeLessThan($windows[0][0] + 5); + + cleanupDirectory(dirname($path)); + }); + + it('never stalls when the overlap is set as large as the window', function (): void { + $path = scratchFile(implode("\n", array_map(fn (int $i): string => "line {$i}", range(1, 30)))); + + $count = 0; + foreach (new LineWindowReader($path, windowLines: 4, overlapLines: 99) as $ignored) { + $count++; + if ($count > 100) { + break; + } + } + + expect($count)->toBeLessThan(100); + + cleanupDirectory(dirname($path)); + }); + + it('yields nothing for an unreadable file rather than throwing', function (): void { + $reader = new LineWindowReader('/no/such/file'); + + expect(iterator_to_array($reader))->toBe([]); + }); + + it('handles a file with no trailing newline', function (): void { + $path = scratchFile('only line, no newline'); + + $windows = iterator_to_array(new LineWindowReader($path, windowLines: 10)); + + expect($windows)->toHaveCount(1) + ->and($windows[0][1])->toBe('only line, no newline'); + + cleanupDirectory(dirname($path)); + }); +}); + +describe('Streaming scan correctness', function (): void { + it('reports the same findings as a single-window scan', function (): void { + $lines = array_map(fn (int $i): string => "line {$i} ordinary text", range(1, 60)); + $lines[9] = 'AWS_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE'; + $lines[39] = 'contact bob@example.com'; + + $path = scratchFile(implode("\n", $lines)); + + $wide = (new Scanner(resolve(Redactor::class), windowLines: 10_000))->scanFile($path, 'file_scan'); + $narrow = (new Scanner(resolve(Redactor::class), windowLines: 7, overlapLines: 3))->scanFile($path, 'file_scan'); + + $shape = fn (ScanResult $result): array => array_map( + fn (ScanFinding $f): string => $f->rule.'@'.$f->line, + $result->findings + ); + + // Windowing must not change what is found or where. + expect($shape($narrow))->toBe($shape($wide)) + ->and($shape($wide))->not->toBeEmpty(); + + cleanupDirectory(dirname($path)); + }); + + it('numbers a finding by its absolute line, not its line within a window', function (): void { + $lines = array_map(fn (int $i): string => "filler {$i}", range(1, 100)); + $lines[74] = 'AWS_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE'; + + $path = scratchFile(implode("\n", $lines)); + + $result = (new Scanner(resolve(Redactor::class), windowLines: 8, overlapLines: 2))->scanFile($path, 'file_scan'); + + $aws = array_values(array_filter($result->findings, fn (ScanFinding $f): bool => $f->rule === 'aws_access_key')); + + expect($aws)->toHaveCount(1) + ->and($aws[0]->line)->toBe(75); + + cleanupDirectory(dirname($path)); + }); + + it('reports a finding in the overlap region only once', function (): void { + $lines = array_map(fn (int $i): string => "filler {$i}", range(1, 20)); + // Line 5 sits inside the overlap of the first two windows. + $lines[4] = 'AWS_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE'; + + $path = scratchFile(implode("\n", $lines)); + + $result = (new Scanner(resolve(Redactor::class), windowLines: 6, overlapLines: 4))->scanFile($path, 'file_scan'); + + $aws = array_filter($result->findings, fn (ScanFinding $f): bool => $f->rule === 'aws_access_key'); + + expect($aws)->toHaveCount(1); + + cleanupDirectory(dirname($path)); + }); + + it('returns findings in file order', function (): void { + $lines = array_map(fn (int $i): string => "filler {$i}", range(1, 40)); + $lines[4] = 'first bob@example.com'; + $lines[24] = 'later AWS_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE'; + + $path = scratchFile(implode("\n", $lines)); + + $result = (new Scanner(resolve(Redactor::class), windowLines: 6, overlapLines: 2))->scanFile($path, 'file_scan'); + + $lineNumbers = array_map(fn (ScanFinding $f): int => $f->line, $result->findings); + $sorted = $lineNumbers; + sort($sorted); + + expect($lineNumbers)->toBe($sorted); + + cleanupDirectory(dirname($path)); + }); + + it('still reports an unreadable file as skipped', function (): void { + $result = (new Scanner(resolve(Redactor::class)))->scanFile('/no/such/file', 'file_scan'); + + expect($result->skipped)->toBeTrue() + ->and($result->error)->toBe('File unreadable'); + }); + + it('finds nothing in a clean file', function (): void { + $path = scratchFile("nothing to see here\njust ordinary prose\nand more of it\n"); + + expect((new Scanner(resolve(Redactor::class)))->scanFile($path, 'file_scan')->hasFindings())->toBeFalse(); + + cleanupDirectory(dirname($path)); + }); +}); + +describe('Streaming memory', function (): void { + it('holds memory flat as the file grows', function (): void { + // A 12 MB file scanned in windows should not cost anything like 12 MB. + $dir = sys_get_temp_dir().'/redactor_big_'.uniqid(); + mkdir($dir); + $path = $dir.'/big.log'; + + $handle = fopen($path, 'wb'); + $line = str_repeat('ordinary log content that is not sensitive ', 4)."\n"; + for ($i = 0; $i < 60_000; $i++) { + fwrite($handle, $line); + } + fwrite($handle, "AWS_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE\n"); + fclose($handle); + + expect(filesize($path))->toBeGreaterThan(10_000_000); + + gc_collect_cycles(); + $before = memory_get_usage(); + + $result = (new Scanner(resolve(Redactor::class)))->scanFile($path, 'file_scan'); + + $growthMb = (memory_get_usage() - $before) / 1_048_576; + + expect($result->hasFindings())->toBeTrue() + // Loading the file whole would be 12 MB before any redaction. + ->and($growthMb)->toBeLessThan(6.0); + + cleanupDirectory($dir); + }); +}); diff --git a/tests/Feature/RedactorTokenizationTest.php b/tests/Feature/RedactorTokenizationTest.php new file mode 100644 index 0000000..4ec178e --- /dev/null +++ b/tests/Feature/RedactorTokenizationTest.php @@ -0,0 +1,186 @@ +set('app.key', 'base64:'.base64_encode(random_bytes(32))); + config()->set('redactor.pseudonymization.key', testPseudonymizationKey()); + config()->set('redactor.profiles.ai', [ + 'enabled' => true, + 'strategies' => [BlockedKeysStrategy::class, RegexPatternsStrategy::class], + 'safe_keys' => [], + 'blocked_keys' => ['ssn'], + 'patterns' => [ + 'email' => ['pattern' => '/[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+/', 'entity' => 'email'], + 'card' => ['pattern' => '/\b\d{16}\b/', 'validator' => 'luhn', 'entity' => 'credit_card'], + ], + 'operators' => ['default' => 'tokenize'], + 'replacement' => '[REDACTED]', + 'mark_redacted' => false, + 'track_redacted_keys' => false, + 'non_redactable_object_behavior' => 'preserve', + 'max_value_length' => null, + 'redact_large_objects' => false, + 'max_object_size' => 100, + 'shannon_entropy' => ['enabled' => false], + ]); + }); + + it('replaces a value with a stable, model-friendly token', function (): void { + $a = Redactor::redact('write to alice@customer.com today', 'ai'); + $b = Redactor::redact('alice@customer.com again', 'ai'); + + expect($a)->toMatch('/^write to tok_email_[a-z0-9]{12} today$/'); + + preg_match('/tok_email_[a-z0-9]{12}/', $a, $ma); + preg_match('/tok_email_[a-z0-9]{12}/', $b, $mb); + + expect($ma[0])->toBe($mb[0]); + }); + + it('round-trips through detokenize, in strings and nested arrays', function (): void { + $prompt = Redactor::redact(['user' => ['ssn' => '123-45-6789'], 'text' => 'mail alice@customer.com card 4111111111111111'], 'ai'); + + expect($prompt['user']['ssn'])->toStartWith('tok_ssn_') + ->and($prompt['text'])->not->toContain('alice@customer.com'); + + $answer = "Reply to {$prompt['text']} and file under {$prompt['user']['ssn']}."; + + expect(Redactor::detokenize($answer)) + ->toBe('Reply to mail alice@customer.com card 4111111111111111 and file under 123-45-6789.') + ->and(Redactor::detokenize($prompt)) + ->toBe(['user' => ['ssn' => '123-45-6789'], 'text' => 'mail alice@customer.com card 4111111111111111']); + }); + + it('leaves a token it does not know exactly as it is', function (): void { + expect(Redactor::detokenize('see tok_email_zzzzzzzzzzzz and tok_made_up_by_model_abcdefghijkl')) + ->toBe('see tok_email_zzzzzzzzzzzz and tok_made_up_by_model_abcdefghijkl'); + }); + + it('keeps the original encrypted in the cache and forgets it on demand', function (): void { + Redactor::redact('alice@customer.com', 'ai'); + + $keys = []; + foreach (Cache::getStore()->all() ?? [] as $k => $v) { + $keys[$k] = $v; + } + + $store = resolve(TokenStore::class); + $token = (new Detokenizer($store))->tokensIn(Redactor::redact('alice@customer.com', 'ai'))[0]; + + expect($store->get($token))->toBe('alice@customer.com') + ->and(Cache::get('redactor:token:'.$token))->not->toContain('alice@customer.com'); + + $store->forget($token); + + expect($store->get($token))->toBeNull() + ->and(Redactor::detokenize($token))->toBe($token); + }); + + it('falls back to plain redaction when no pseudonymization key is available', function (): void { + config()->set('redactor.pseudonymization', ['enabled' => false]); + + expect(Redactor::redact('alice@customer.com', 'ai'))->toBe('[REDACTED]'); + }); + + it('honours a per-entity ttl option', function (): void { + config()->set('redactor.profiles.ai.operators', ['default' => 'redact', 'email' => ['tokenize' => ['ttl' => 5]]]); + + $out = Redactor::redact('alice@customer.com', 'ai'); + $token = (new Detokenizer(resolve(TokenStore::class)))->tokensIn($out)[0]; + + expect(resolve(TokenStore::class)->get($token))->toBe('alice@customer.com'); + + $this->travel(6)->seconds(); + + expect(resolve(TokenStore::class)->get($token))->toBeNull(); + }); + + it('does not resolve the cache until something is tokenised', function (): void { + $redactor = resolve(RedactorService::class); + + expect($redactor->operators()->has('tokenize'))->toBeTrue() + ->and($redactor->redact('nothing sensitive'))->toBe('nothing sensitive'); + }); +}); + +describe('The token store', function (): void { + beforeEach(function (): void { + config()->set('app.key', 'base64:'.base64_encode(random_bytes(32))); + config()->set('redactor.pseudonymization.key', testPseudonymizationKey()); + config()->set('redactor.profiles.ai', [ + 'enabled' => true, + 'strategies' => [RegexPatternsStrategy::class], + 'safe_keys' => [], + 'blocked_keys' => [], + 'patterns' => ['email' => ['pattern' => '/[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+/', 'entity' => 'email']], + 'operators' => ['default' => 'tokenize'], + 'replacement' => '[REDACTED]', + 'mark_redacted' => false, + 'track_redacted_keys' => false, + 'non_redactable_object_behavior' => 'preserve', + 'max_value_length' => null, + 'redact_large_objects' => false, + 'max_object_size' => 100, + 'shannon_entropy' => ['enabled' => false], + ]); + }); + + it('keeps a token for good when no ttl is configured', function (): void { + config()->set('redactor.tokenization.ttl'); + + $token = (new Detokenizer(resolve(TokenStore::class)))->tokensIn(Redactor::redact('alice@customer.com', 'ai'))[0]; + + $this->travel(10)->years(); + + expect(resolve(TokenStore::class)->get($token))->toBe('alice@customer.com'); + }); + + it('treats a token whose payload no longer decrypts as unknown', function (): void { + $out = Redactor::redact('alice@customer.com', 'ai'); + $token = (new Detokenizer(resolve(TokenStore::class)))->tokensIn($out)[0]; + + Cache::put('redactor:token:'.$token, 'not-a-ciphertext', 60); + + expect(resolve(TokenStore::class)->get($token))->toBeNull() + ->and(Redactor::detokenize($out))->toBe($out); + }); + + it('leaves content that is neither text nor an array alone when detokenizing', function (): void { + expect(Redactor::detokenize(42))->toBe(42) + ->and(Redactor::detokenize(null))->toBeNull(); + }); + + it('resolves the underlying store once, on first use, for reads and forgets alike', function (): void { + $inner = resolve(TokenStore::class); + $resolved = 0; + $lazy = new LazyTokenStore(function () use ($inner, &$resolved): TokenStore { + $resolved++; + + return $inner; + }); + + expect($resolved)->toBe(0); + + $lazy->put('tok_email_abcdefghijkl', 'alice@customer.com', 'email'); + + expect($lazy->get('tok_email_abcdefghijkl'))->toBe('alice@customer.com'); + + $lazy->forget('tok_email_abcdefghijkl'); + + expect($lazy->get('tok_email_abcdefghijkl'))->toBeNull() + ->and($resolved)->toBe(1); + }); +}); diff --git a/tests/Feature/RedactorValidatorTest.php b/tests/Feature/RedactorValidatorTest.php new file mode 100644 index 0000000..5e31413 --- /dev/null +++ b/tests/Feature/RedactorValidatorTest.php @@ -0,0 +1,201 @@ + true, + 'strategies' => [RegexPatternsStrategy::class], + 'safe_keys' => [], + 'blocked_keys' => [], + 'patterns' => $patterns, + 'replacement' => '[REDACTED]', + 'mark_redacted' => false, + 'track_redacted_keys' => false, + 'non_redactable_object_behavior' => 'preserve', + 'max_value_length' => null, + 'redact_large_objects' => false, + 'max_object_size' => 100, + 'shannon_entropy' => ['enabled' => false], + ]; +} + +describe('Luhn', function (): void { + it('accepts real card numbers', function (): void { + foreach ([ + '4111111111111111', // Visa test + '5500005555555559', // Mastercard test + '378282246310005', // Amex test + '6011111111111117', // Discover test + '4111 1111 1111 1111', + '4111-1111-1111-1111', + ] as $card) { + expect(Validator::luhn($card))->toBeTrue("expected {$card} to pass Luhn"); + } + }); + + it('rejects numbers of the right shape that are not cards', function (): void { + foreach ([ + '4111111111111112', // one digit off + '1234567890123456', + '2024010112000001', // concatenated timestamp + '9999999999999999', + ] as $notCard) { + expect(Validator::luhn($notCard))->toBeFalse("expected {$notCard} to fail Luhn"); + } + }); + + it('rejects runs that are too short or too long to be a card', function (): void { + expect(Validator::luhn('12345678901'))->toBeFalse() + ->and(Validator::luhn('12345678901234567890'))->toBeFalse(); + }); +}); + +describe('IBAN', function (): void { + it('accepts valid IBANs', function (): void { + foreach ([ + 'GB82WEST12345698765432', + 'DE89370400440532013000', + 'FR1420041010050500013M02606', + 'GB82 WEST 1234 5698 7654 32', + ] as $iban) { + expect(Validator::iban($iban))->toBeTrue("expected {$iban} to pass mod-97"); + } + }); + + it('rejects a wrong check digit', function (): void { + expect(Validator::iban('GB82WEST12345698765431'))->toBeFalse() + ->and(Validator::iban('DE89370400440532013001'))->toBeFalse(); + }); + + it('rejects malformed input', function (): void { + expect(Validator::iban('12345678901234567'))->toBeFalse() + ->and(Validator::iban('GB'))->toBeFalse(); + }); +}); + +describe('SSN', function (): void { + it('accepts issuable numbers', function (): void { + expect(Validator::ssn('123-45-6789'))->toBeTrue() + ->and(Validator::ssn('123456789'))->toBeTrue(); + }); + + it('rejects never-issued area, group and serial values', function (): void { + expect(Validator::ssn('000-45-6789'))->toBeFalse() + ->and(Validator::ssn('666-45-6789'))->toBeFalse() + ->and(Validator::ssn('900-45-6789'))->toBeFalse() + ->and(Validator::ssn('123-00-6789'))->toBeFalse() + ->and(Validator::ssn('123-45-0000'))->toBeFalse(); + }); + + it('rejects the wrong number of digits', function (): void { + expect(Validator::ssn('12345678'))->toBeFalse() + ->and(Validator::ssn('1234567890'))->toBeFalse(); + }); +}); + +describe('Validators inside redaction', function (): void { + it('redacts a valid card and leaves an invalid lookalike alone', function (): void { + config()->set('redactor.profiles.validated', validatorProfile([ + 'card' => [ + 'pattern' => '/\b(?:\d[ -]*?){13,16}\b/', + 'validator' => 'luhn', + 'mode' => 'partial', + 'keep' => 4, + ], + ])); + + expect(resolve(Redactor::class)->redact('card 4111111111111111 ok', 'validated')) + ->toBe('card ************1111 ok'); + + // An order number of the same shape survives. + expect(resolve(Redactor::class)->redact('order 2024010112000001 ok', 'validated')) + ->toBe('order 2024010112000001 ok'); + }); + + it('does not mark a payload redacted when every match failed validation', function (): void { + config()->set('redactor.profiles.validated', validatorProfile([ + 'card' => ['pattern' => '/\b\d{16}\b/', 'validator' => 'luhn'], + ])); + + $result = resolve(Redactor::class)->inspect(['n' => '1234567890123456'], 'validated'); + + expect($result->wasRedacted)->toBeFalse() + ->and($result->value)->toBe(['n' => '1234567890123456']); + }); + + it('redacts the valid matches and leaves the invalid ones in the same string', function (): void { + config()->set('redactor.profiles.validated', validatorProfile([ + 'card' => ['pattern' => '/\b\d{16}\b/', 'validator' => 'luhn'], + ])); + + expect(resolve(Redactor::class)->redact('good 4111111111111111 bad 1234567890123456', 'validated')) + ->toBe('good [REDACTED] bad 1234567890123456'); + }); + + it('validates the capture group, not the surrounding context', function (): void { + config()->set('redactor.profiles.validated', validatorProfile([ + 'card' => ['pattern' => '/(card:\s*)(\d{16})/', 'capture' => 2, 'validator' => 'luhn'], + ])); + + expect(resolve(Redactor::class)->redact('card: 4111111111111111', 'validated')) + ->toBe('card: [REDACTED]') + ->and(resolve(Redactor::class)->redact('card: 1234567890123456', 'validated')) + ->toBe('card: 1234567890123456'); + }); + + it('applies validation in full mode too', function (): void { + config()->set('redactor.profiles.validated', validatorProfile([ + 'card' => ['pattern' => '/\b\d{16}\b/', 'validator' => 'luhn', 'mode' => 'full'], + ])); + + expect(resolve(Redactor::class)->redact('n 1234567890123456', 'validated')) + ->toBe('n 1234567890123456') + ->and(resolve(Redactor::class)->redact('n 4111111111111111', 'validated')) + ->toBe('[REDACTED]'); + }); + + it('rejects an unknown validator name in config', function (): void { + config()->set('redactor.profiles.validated', validatorProfile([ + 'card' => ['pattern' => '/\d+/', 'validator' => 'vibes'], + ])); + + expect(fn (): RedactorConfig => RedactorConfig::fromConfig('validated')) + ->toThrow(\InvalidArgumentException::class, 'patterns.card.validator'); + }); +}); + +describe('Shipped profiles use validators', function (): void { + it('no longer flags an order number as a credit card', function (): void { + $result = resolve(Redactor::class)->redact(['note' => 'order 2024010112000001 shipped'], 'default'); + + expect($result['note'])->toBe('order 2024010112000001 shipped'); + }); + + it('still redacts a real card in the default profile', function (): void { + $result = resolve(Redactor::class)->redact(['note' => 'paid with 4111111111111111'], 'default'); + + expect($result['note'])->not->toContain('4111111111111111') + ->and($result['note'])->toContain('1111'); + }); + + it('no longer flags 000-00-0000 as an SSN', function (): void { + $result = resolve(Redactor::class)->redact(['note' => 'placeholder 000-00-0000'], 'default'); + + expect($result['note'])->toBe('placeholder 000-00-0000'); + }); + + it('redacts a valid IBAN', function (): void { + $result = resolve(Redactor::class)->redact(['note' => 'pay GB82WEST12345698765432 now'], 'default'); + + expect($result['note'])->not->toContain('GB82WEST12345698765432'); + }); +}); diff --git a/tests/Feature/RedactorVerificationTest.php b/tests/Feature/RedactorVerificationTest.php new file mode 100644 index 0000000..c7800fe --- /dev/null +++ b/tests/Feature/RedactorVerificationTest.php @@ -0,0 +1,363 @@ + */ + public static array $seen = []; + + public function __construct( + private readonly VerificationStatus $status = VerificationStatus::Active, + ) {} + + public function name(): string + { + return 'spy'; + } + + public function host(): string + { + return 'spy.invalid'; + } + + public function supports(string $entity, string $rule): bool + { + return true; + } + + public function verify(string $secret): VerificationResult + { + self::$seen[] = $secret; + + return match ($this->status) { + VerificationStatus::Active => VerificationResult::active(), + VerificationStatus::Inactive => VerificationResult::inactive(), + VerificationStatus::Unknown => VerificationResult::unknown(), + }; + } +} + +function secretFile(string $contents): string +{ + $dir = sys_get_temp_dir().'/redactor_verify_'.uniqid(); + mkdir($dir); + file_put_contents($dir.'/app.env', $contents); + + return $dir.'/app.env'; +} + +describe('Verification is off unless three things agree', function (): void { + it('stays off when config does not enable it', function (): void { + expect(SecretVerifier::fromConfig(['enabled' => false, 'verifiers' => ['github_token']]))->toBeNull(); + }); + + it('stays off when enabled but no provider is allowed', function (): void { + // Enabling the feature and choosing who to trust are separate + // decisions; an empty list means none, not all. + expect(SecretVerifier::fromConfig(['enabled' => true, 'verifiers' => []]))->toBeNull() + ->and(SecretVerifier::fromConfig(['enabled' => true]))->toBeNull(); + }); + + it('runs only the providers on the allowlist', function (): void { + $verifier = new SecretVerifier(['github_token']); + + $names = array_map(fn (Verifier $v): string => $v->name(), $verifier->enabled()); + + expect($names)->toBe(['github_token']) + ->and($verifier->canVerify('github_token', 'github_token'))->toBeTrue() + ->and($verifier->canVerify('stripe_key', 'api_key_stripe'))->toBeFalse(); + }); + + it('names every host it would contact', function (): void { + $verifier = new SecretVerifier(['github_token', 'stripe_key', 'slack_token']); + + expect($verifier->hosts())->toBe(['api.github.com', 'api.stripe.com', 'slack.com']); + }); + + it('reports Unknown rather than silently skipping an unsupported entity', function (): void { + $result = (new SecretVerifier(['github_token']))->verify('stripe_key', 'api_key_stripe', 'sk_live_x'); + + expect($result->status)->toBe(VerificationStatus::Unknown) + ->and($result->note)->toContain('No verifier is enabled'); + }); + + it('degrades to Unknown when a verifier throws', function (): void { + $exploding = new class implements Verifier + { + public function name(): string + { + return 'boom'; + } + + public function host(): string + { + return 'boom.invalid'; + } + + public function supports(string $e, string $r): bool + { + return true; + } + + public function verify(string $s): VerificationResult + { + throw new \RuntimeException('network on fire'); + } + }; + + $result = (new SecretVerifier(['boom'], [$exploding]))->verify('x', 'y', 'secret'); + + expect($result->status)->toBe(VerificationStatus::Unknown) + ->and($result->note)->toContain('network on fire'); + }); +}); + +describe('Verification never leaks the secret', function (): void { + afterEach(fn (): array => SpyVerifier::$seen = []); + + it('keeps the secret out of the finding and its output', function (): void { + SpyVerifier::$seen = []; + + $path = secretFile("GITHUB_TOKEN=ghp_abcdefghijklmnopqrstuvwxyz0123456789\n"); + + $scanner = (new Scanner(resolve(Redactor::class))) + ->withVerifier(new SecretVerifier(['spy'], [new SpyVerifier])); + + $result = $scanner->scanFile($path, 'file_scan'); + + $encoded = json_encode(array_map(fn (ScanFinding $f): array => $f->toArray(), $result->findings)); + + // The verifier saw it - that is its job - but nothing that gets written + // out did. + expect(SpyVerifier::$seen)->not->toBeEmpty() + ->and($encoded)->not->toContain('ghp_abcdefghijklmnopqrstuvwxyz0123456789'); + + cleanupDirectory(dirname($path)); + }); + + it('sends nothing at all when no verifier is attached', function (): void { + SpyVerifier::$seen = []; + + $path = secretFile("GITHUB_TOKEN=ghp_abcdefghijklmnopqrstuvwxyz0123456789\n"); + + (new Scanner(resolve(Redactor::class)))->scanFile($path, 'file_scan'); + + expect(SpyVerifier::$seen)->toBe([]); + + cleanupDirectory(dirname($path)); + }); +}); + +describe('Verification changes triage', function (): void { + afterEach(fn (): array => SpyVerifier::$seen = []); + + it('ranks a confirmed-live credential above everything else', function (): void { + $path = secretFile("GITHUB_TOKEN=ghp_abcdefghijklmnopqrstuvwxyz0123456789\n"); + + $scanner = (new Scanner(resolve(Redactor::class))) + ->withVerifier(new SecretVerifier(['spy'], [new SpyVerifier(VerificationStatus::Active)])); + + $finding = $scanner->scanFile($path, 'file_scan')->findings[0]; + + expect($finding->severity())->toBe('critical') + ->and($finding->verification?->status)->toBe(VerificationStatus::Active); + + cleanupDirectory(dirname($path)); + }); + + it('does not downgrade an unverifiable finding to safe', function (): void { + // A check that could not complete is not evidence of safety. + expect(VerificationStatus::Unknown->severity())->toBe('high') + ->and(VerificationStatus::Inactive->severity())->toBe('low') + ->and(VerificationStatus::Active->severity())->toBe('critical'); + }); + + it('reports the verdict in JSON output', function (): void { + $path = secretFile("GITHUB_TOKEN=ghp_abcdefghijklmnopqrstuvwxyz0123456789\n"); + + $scanner = (new Scanner(resolve(Redactor::class))) + ->withVerifier(new SecretVerifier(['spy'], [new SpyVerifier(VerificationStatus::Inactive)])); + + $finding = $scanner->scanFile($path, 'file_scan')->findings[0]->toArray(); + + expect($finding['verification']['status'])->toBe('inactive') + ->and($finding['verification']['verifier'])->toBe('spy'); + + cleanupDirectory(dirname($path)); + }); +}); + +describe('Built-in verifiers', function (): void { + it('reads GitHub 401 as inactive', function (): void { + Http::fake(['api.github.com/*' => Http::response([], 401)]); + + expect((new GitHubTokenVerifier)->verify('ghp_x')->status)->toBe(VerificationStatus::Inactive); + }); + + it('reads GitHub 200 as live', function (): void { + Http::fake(['api.github.com/*' => Http::response(['login' => 'someone'], 200)]); + + expect((new GitHubTokenVerifier)->verify('ghp_x')->status)->toBe(VerificationStatus::Active); + }); + + it('reads an unexpected GitHub status as unknown', function (): void { + Http::fake(['api.github.com/*' => Http::response([], 503)]); + + expect((new GitHubTokenVerifier)->verify('ghp_x')->status)->toBe(VerificationStatus::Unknown); + }); + + it('reads Stripe 401 as inactive', function (): void { + Http::fake(['api.stripe.com/*' => Http::response([], 401)]); + + expect((new StripeKeyVerifier)->verify('sk_live_x')->status)->toBe(VerificationStatus::Inactive); + }); + + it('reads Stripe 200 as live', function (): void { + Http::fake(['api.stripe.com/*' => Http::response(['object' => 'balance'], 200)]); + + expect((new StripeKeyVerifier)->verify('sk_live_x')->status)->toBe(VerificationStatus::Active); + }); + + it('reads a Slack rejection from the body, not the status code', function (): void { + // Slack answers 200 either way; trusting the status alone would call + // every dead token live. + Http::fake(['slack.com/*' => Http::response(['ok' => false, 'error' => 'invalid_auth'], 200)]); + + $result = (new SlackTokenVerifier)->verify('xoxb-x'); + + expect($result->status)->toBe(VerificationStatus::Inactive) + ->and($result->note)->toContain('invalid_auth'); + }); + + it('reads a Slack acceptance from the body', function (): void { + Http::fake(['slack.com/*' => Http::response(['ok' => true, 'team' => 'acme'], 200)]); + + expect((new SlackTokenVerifier)->verify('xoxb-x')->status)->toBe(VerificationStatus::Active); + }); + + it('never lets a transport failure escape as an exception', function (): void { + Http::fake(fn () => throw new \RuntimeException('connection refused')); + + expect((new GitHubTokenVerifier)->verify('ghp_x')->status)->toBe(VerificationStatus::Unknown) + ->and((new StripeKeyVerifier)->verify('sk_x')->status)->toBe(VerificationStatus::Unknown) + ->and((new SlackTokenVerifier)->verify('xoxb-x')->status)->toBe(VerificationStatus::Unknown); + }); + + it('routes each entity to the right verifier', function (): void { + expect((new GitHubTokenVerifier)->supports('github_token', 'x'))->toBeTrue() + ->and((new GitHubTokenVerifier)->supports('stripe_key', 'x'))->toBeFalse() + ->and((new StripeKeyVerifier)->supports('x', 'api_key_stripe'))->toBeTrue() + ->and((new SlackTokenVerifier)->supports('slack_token', 'x'))->toBeTrue(); + }); +}); + +describe('The scan command gate', function (): void { + beforeEach(function (): void { + config(['redactor.scan.profile' => 'file_scan', 'redactor.scan.baseline' => null]); + $this->path = secretFile("GITHUB_TOKEN=ghp_abcdefghijklmnopqrstuvwxyz0123456789\n"); + }); + + afterEach(fn () => cleanupDirectory(dirname($this->path))); + + it('refuses --verify when config has not enabled it', function (): void { + config(['redactor.scan.verification' => ['enabled' => false, 'verifiers' => ['github_token']]]); + + $exit = Artisan::call('redactor:scan', ['paths' => [$this->path], '--verify' => true]); + + expect($exit)->toBe(1) + ->and(Artisan::output())->toContain('Verification is not enabled'); + }); + + it('refuses --verify when enabled with an empty allowlist', function (): void { + config(['redactor.scan.verification' => ['enabled' => true, 'verifiers' => []]]); + + expect(Artisan::call('redactor:scan', ['paths' => [$this->path], '--verify' => true]))->toBe(1); + }); + + it('names the hosts before contacting any of them', function (): void { + config(['redactor.scan.verification' => ['enabled' => true, 'verifiers' => ['github_token']]]); + Http::fake(['api.github.com/*' => Http::response([], 401)]); + + Artisan::call('redactor:scan', ['paths' => [$this->path], '--verify' => true]); + + expect(Artisan::output())->toContain('api.github.com'); + }); + + it('sends nothing when --verify is absent, however config is set', function (): void { + config(['redactor.scan.verification' => ['enabled' => true, 'verifiers' => ['github_token']]]); + Http::fake(); + + Artisan::call('redactor:scan', ['paths' => [$this->path]]); + + Http::assertNothingSent(); + }); +}); + +describe('Built-in verifiers on unexpected answers', function (): void { + it('reads a non-2xx from Slack, a Slack body without ok, and a non-401 failure from Stripe as unknown', function (): void { + Http::fake([ + 'slack.com/*' => Http::sequence()->push([], 500)->push(['team' => 'acme'], 200), + 'api.stripe.com/*' => Http::response([], 503), + ]); + + $slackDown = (new SlackTokenVerifier)->verify('xoxb-x'); + $slackOdd = (new SlackTokenVerifier)->verify('xoxb-x'); + $stripe = (new StripeKeyVerifier)->verify('sk_live_x'); + + expect($slackDown->status)->toBe(VerificationStatus::Unknown) + ->and($slackDown->note)->toContain('Slack returned 500') + ->and($slackOdd->status)->toBe(VerificationStatus::Unknown) + ->and($slackOdd->note)->toContain('unexpected') + ->and($stripe->status)->toBe(VerificationStatus::Unknown) + ->and($stripe->note)->toContain('Stripe returned 503'); + }); + + it('leaves a finding unverified when no enabled verifier understands its entity', function (): void { + Http::fake(); + $path = secretFile("STRIPE_KEY=sk_live_4eC39HqLyjWDarjtT1zdp7dc\n"); + + $scanner = (new Scanner(resolve(Redactor::class))) + ->withVerifier(new SecretVerifier(['github_token'], [new GitHubTokenVerifier])); + + $result = $scanner->scanFile($path, 'file_scan'); + + expect($result->findings)->not->toBeEmpty() + ->and($result->findings[0]->verification)->toBeNull(); + + Http::assertNothingSent(); + + cleanupDirectory(dirname($path)); + }); + + it('marks a confirmed-live credential LIVE in the table', function (): void { + config(['redactor.scan.profile' => 'file_scan', 'redactor.scan.baseline' => null]); + config(['redactor.scan.verification' => ['enabled' => true, 'verifiers' => ['github_token']]]); + Http::fake(['api.github.com/*' => Http::response(['login' => 'someone'], 200)]); + $path = secretFile("GITHUB_TOKEN=ghp_abcdefghijklmnopqrstuvwxyz0123456789\n"); + + Artisan::call('redactor:scan', ['paths' => [$path], '--verify' => true]); + + expect(Artisan::output())->toContain('LIVE'); + + cleanupDirectory(dirname($path)); + }); +}); diff --git a/tests/Feature/RedactorVerifiersTest.php b/tests/Feature/RedactorVerifiersTest.php new file mode 100644 index 0000000..6d564db --- /dev/null +++ b/tests/Feature/RedactorVerifiersTest.php @@ -0,0 +1,120 @@ + Http::sequence()->push(['data' => []], 200)->push([], 401)->push([], 503)]); + + expect($verifier->verify('sk-x')->status)->toBe(VerificationStatus::Active) + ->and($verifier->verify('sk-x')->status)->toBe(VerificationStatus::Inactive) + ->and($verifier->verify('sk-x')->status)->toBe(VerificationStatus::Unknown); + + expect($verifier->supports('openai_key', 'x'))->toBeTrue() + ->and($verifier->supports('other', 'my_openai_rule'))->toBeTrue() + ->and($verifier->supports('other', 'x'))->toBeFalse() + ->and($verifier->host())->toBe('api.openai.com'); + }); + + it('classify Anthropic answers and send the version header', function (): void { + $verifier = new AnthropicKeyVerifier; + + Http::fake(['api.anthropic.com/*' => Http::sequence()->push(['data' => []], 200)->push([], 401)->push([], 529)]); + + expect($verifier->verify('sk-ant-x')->status)->toBe(VerificationStatus::Active) + ->and($verifier->verify('sk-ant-x')->status)->toBe(VerificationStatus::Inactive) + ->and($verifier->verify('sk-ant-x')->status)->toBe(VerificationStatus::Unknown); + + Http::assertSent(fn ($request): bool => $request->hasHeader('x-api-key', 'sk-ant-x') && $request->hasHeader('anthropic-version')); + + expect($verifier->name())->toBe('anthropic_key') + ->and($verifier->supports('anthropic_key', 'x'))->toBeTrue(); + }); + + it('classify SendGrid answers', function (): void { + $verifier = new SendGridKeyVerifier; + + Http::fake(['api.sendgrid.com/*' => Http::sequence()->push(['scopes' => []], 200)->push([], 403)->push([], 401)->push([], 500)]); + + expect($verifier->verify('SG.x')->status)->toBe(VerificationStatus::Active) + ->and($verifier->verify('SG.x')->status)->toBe(VerificationStatus::Inactive) + ->and($verifier->verify('SG.x')->status)->toBe(VerificationStatus::Inactive) + ->and($verifier->verify('SG.x')->status)->toBe(VerificationStatus::Unknown); + + expect($verifier->name())->toBe('sendgrid_key') + ->and($verifier->supports('sendgrid_key', 'x'))->toBeTrue(); + }); + + it('classify Google answers, treating a disabled-API 403 as live', function (): void { + $verifier = new GoogleApiKeyVerifier; + + Http::fake(['generativelanguage.googleapis.com/*' => Http::sequence() + ->push(['models' => []], 200) + ->push([], 403) + ->push(['error' => 'API key not valid'], 400) + ->push([], 502)]); + + expect($verifier->verify('AIzaX')->status)->toBe(VerificationStatus::Active) + ->and($verifier->verify('AIzaX')->status)->toBe(VerificationStatus::Active) + ->and($verifier->verify('AIzaX')->status)->toBe(VerificationStatus::Inactive) + ->and($verifier->verify('AIzaX')->status)->toBe(VerificationStatus::Unknown); + + Http::assertSent(fn ($request): bool => str_contains($request->url(), 'key=AIzaX')); + + expect($verifier->name())->toBe('google_api_key') + ->and($verifier->supports('google_api_key', 'x'))->toBeTrue(); + }); + + it('report an unreachable provider as unknown', function (): void { + Http::fake(fn () => throw new \RuntimeException('dns')); + + foreach ([new OpenAiKeyVerifier, new AnthropicKeyVerifier, new SendGridKeyVerifier, new GoogleApiKeyVerifier] as $verifier) { + expect($verifier->verify('x')->status)->toBe(VerificationStatus::Unknown); + } + }); + + it('are all shipped and allow-listable by name, alongside registered ones', function (): void { + SecretVerifier::register(new class implements Verifier + { + public function name(): string + { + return 'acme'; + } + + public function host(): string + { + return 'acme.test'; + } + + public function supports(string $entity, string $rule): bool + { + return $entity === 'acme_key'; + } + + public function verify(string $secret): VerificationResult + { + return VerificationResult::active(); + } + }); + + $verifier = new SecretVerifier(['openai_key', 'anthropic_key', 'sendgrid_key', 'google_api_key', 'acme']); + + expect($verifier->hosts())->toBe(['acme.test', 'api.anthropic.com', 'api.openai.com', 'api.sendgrid.com', 'generativelanguage.googleapis.com']) + ->and($verifier->canVerify('acme_key', 'x'))->toBeTrue() + ->and($verifier->verify('acme_key', 'x', 's')->status)->toBe(VerificationStatus::Active); + }); +}); diff --git a/tests/Feature/RedactorWildcardTest.php b/tests/Feature/RedactorWildcardTest.php index cb3f52d..d46b71c 100644 --- a/tests/Feature/RedactorWildcardTest.php +++ b/tests/Feature/RedactorWildcardTest.php @@ -5,15 +5,16 @@ namespace Tests\Feature; use Kirschbaum\Redactor\Redactor; +use Kirschbaum\Redactor\Strategies\BlockedKeysStrategy; -describe('Redactor Wildcard Blocked Keys Tests', function () { - it('matches wildcard patterns for blocked keys', function () { +describe('Redactor Wildcard Blocked Keys Tests', function (): void { + it('matches wildcard patterns for blocked keys', function (): void { // Configure a test profile with wildcard patterns config()->set('redactor.default_profile', 'wildcard_test'); config()->set('redactor.profiles.wildcard_test', [ 'enabled' => true, 'strategies' => [ - \Kirschbaum\Redactor\Strategies\BlockedKeysStrategy::class, + BlockedKeysStrategy::class, ], 'safe_keys' => [], 'blocked_keys' => ['*token*', '*key*', 'password'], @@ -62,12 +63,12 @@ ->and($result['_redacted'])->toBeTrue(); }); - it('supports exact matches alongside wildcard patterns', function () { + it('supports exact matches alongside wildcard patterns', function (): void { config()->set('redactor.default_profile', 'wildcard_test'); config()->set('redactor.profiles.wildcard_test', [ 'enabled' => true, 'strategies' => [ - \Kirschbaum\Redactor\Strategies\BlockedKeysStrategy::class, + BlockedKeysStrategy::class, ], 'safe_keys' => [], 'blocked_keys' => ['exact_match', '*partial*'], @@ -103,12 +104,12 @@ ->and($result['other_field'])->toBe('should_stay'); }); - it('handles case-insensitive wildcard matching', function () { + it('handles case-insensitive wildcard matching', function (): void { config()->set('redactor.default_profile', 'wildcard_test'); config()->set('redactor.profiles.wildcard_test', [ 'enabled' => true, 'strategies' => [ - \Kirschbaum\Redactor\Strategies\BlockedKeysStrategy::class, + BlockedKeysStrategy::class, ], 'safe_keys' => [], 'blocked_keys' => ['*TOKEN*'], @@ -142,12 +143,12 @@ ->and($result['other_field'])->toBe('should_stay'); }); - it('supports multiple wildcard positions', function () { + it('supports multiple wildcard positions', function (): void { config()->set('redactor.default_profile', 'wildcard_test'); config()->set('redactor.profiles.wildcard_test', [ 'enabled' => true, 'strategies' => [ - \Kirschbaum\Redactor\Strategies\BlockedKeysStrategy::class, + BlockedKeysStrategy::class, ], 'safe_keys' => [], 'blocked_keys' => ['user_*_token', '*_key_*'], diff --git a/tests/Feature/StreamRedactorTest.php b/tests/Feature/StreamRedactorTest.php new file mode 100644 index 0000000..3d97106 --- /dev/null +++ b/tests/Feature/StreamRedactorTest.php @@ -0,0 +1,150 @@ +push($chunk); + } + + $emitted[] = $stream->flush(); + + return $emitted; +} + +describe('StreamRedactor', function (): void { + it('catches a secret split across two chunks', function (): void { + $text = "log line one\nthe key is sk_live_4eC39HqLyjWDarjtT1zdp7dc ok\nline three\n"; + $split = strpos($text, 'sk_live_') + 12; // mid-token + + $emitted = streamed([substr($text, 0, $split), substr($text, $split)], 16); + + expect(implode('', $emitted))->toBe(resolve(Redactor::class)->redact($text, 'file_scan')) + ->and(implode('', $emitted))->not->toContain('4eC39HqLyjWDarjtT1zdp7dc'); + }); + + it('produces the same output as a whole-string redaction however the chunks fall', function (): void { + $text = str_repeat("user bob@example.com paid with 4111111111111111 on 2026-09-13\n", 40); + $whole = resolve(Redactor::class)->redact($text, 'file_scan'); + + foreach ([1, 7, 64, 1000] as $size) { + expect(implode('', streamed(str_split($text, $size), 48)))->toBe($whole, "chunk size {$size}"); + } + }); + + it('never emits anything that a later chunk could have completed', function (): void { + $stream = new StreamRedactor(resolve(Redactor::class), 'file_scan', 20); + + $first = $stream->push('hello sk_live_4eC39Hq'); + + expect($first)->toBe(''); + + $second = $stream->push("LyjWDarjtT1zdp7dc bye\nmore text to push the window along\n"); + + expect($first.$second.$stream->flush())->not->toContain('4eC39'); + }); + + it('holds an open PEM block whole', function (): void { + $pem = "-----BEGIN RSA PRIVATE KEY-----\nMIIEowIBAAKCAQEA\nMIIEowIBAAKCAQEB\nMIIEowIBAAKCAQEC\n-----END RSA PRIVATE KEY-----\n"; + $text = "header line here to fill the buffer\n".$pem."trailer\n"; + + $out = implode('', streamed(str_split($text, 10), 24)); + + expect($out)->not->toContain('MIIEowIBAAKCAQEA') + ->and($out)->toContain('header line') + ->and($out)->toContain('trailer'); + }); + + it('streams through a generator', function (): void { + $stream = new StreamRedactor(resolve(Redactor::class), 'file_scan', 8); + + $out = implode('', iterator_to_array($stream->through(['mail bob@ex', 'ample.com now and ', 'that is all']))); + + expect($out)->toBe('mail [REDACTED] now and that is all'); + }); + + it('keeps memory bounded by the hold-back, not the stream', function (): void { + $stream = new StreamRedactor(resolve(Redactor::class), 'file_scan', 64); + $chunk = str_repeat("plain words only here\n", 10); + + memory_reset_peak_usage(); + $before = memory_get_peak_usage(); + + for ($i = 0; $i < 2000; $i++) { + $stream->push($chunk); + } + + expect(memory_get_peak_usage() - $before)->toBeLessThan(2 * 1024 * 1024); + }); + + it('wraps an echoing callback and redacts what it echoes', function (): void { + $stream = new StreamRedactor(resolve(Redactor::class), 'file_scan', 16); + + $wrapped = $stream->wrap(function (): void { + echo 'first bob@exam'; + echo "ple.com second\n"; + echo 'AKIAIOSFODNN7EXAMPLE end'; + }, 8); + + ob_start(); + $wrapped(); + $out = ob_get_clean(); + + expect($out)->toBe("first [REDACTED] second\n[REDACTED] end"); + }); +}); + +describe('Streamed responses through the redact middleware', function (): void { + it('redacts a streamed response as it streams', function (): void { + Route::get('/stream', fn () => response()->stream(function (): void { + echo "event: message\ndata: contact bob@exam"; + echo "ple.com and token sk_live_4eC39HqLyjWDarjtT1zdp7dc\n\n"; + }, 200, ['Content-Type' => 'text/event-stream']))->middleware('redact:file_scan'); + + $response = $this->get('/stream'); + + $response->assertOk(); + $content = $response->streamedContent(); + + expect($content)->toContain('data: contact [REDACTED] and token [REDACTED]') + ->and($content)->not->toContain('bob@example.com'); + }); +}); + +describe('StreamRedactor cut points and responses', function (): void { + it('emits an unbroken run once it has outlived the hold-back window', function (): void { + $stream = new StreamRedactor(resolve(Redactor::class), 'file_scan', 8); + + expect($stream->push(str_repeat('a', 40)))->toBe(str_repeat('a', 32)) + ->and($stream->flush())->toBe(str_repeat('a', 8)); + }); + + it('builds a streamed response whose output is redacted on the way out', function (): void { + $stream = new StreamRedactor(resolve(Redactor::class), 'file_scan', 16); + + $response = $stream->response(function (): void { + echo "contact bob@example.com\n"; + }, 201, ['X-Test' => 'yes'], 8); + + expect($response)->toBeInstanceOf(StreamedResponse::class) + ->and($response->getStatusCode())->toBe(201) + ->and($response->headers->get('X-Test'))->toBe('yes'); + + ob_start(); + $response->sendContent(); + $out = ob_get_clean(); + + expect($out)->toBe("contact [REDACTED]\n"); + }); +}); diff --git a/tests/Performance/HotPathTest.php b/tests/Performance/HotPathTest.php new file mode 100644 index 0000000..bb6f4cd --- /dev/null +++ b/tests/Performance/HotPathTest.php @@ -0,0 +1,123 @@ +safeKeyMatcher)->toBe($second->safeKeyMatcher) + ->and($first->blockedKeyMatcher)->toBe($second->blockedKeyMatcher); + }); + + it('is far cheaper than looking the matcher up per call', function (): void { + $config = RedactorConfig::fromConfig('default'); + $keys = ['user_id', 'password', 'created_at', 'normal_field', 'api_token']; + + $held = fastest(function () use ($config, $keys): void { + foreach ($keys as $key) { + $config->blockedKeyMatcher->matches($key); + } + }, 20_000); + + // What it used to do: find the memoised matcher by rebuilding an + // implode() of every configured key, on every single check. + $lookedUp = fastest(function () use ($config, $keys): void { + foreach ($keys as $key) { + KeyMatcher::for($config->blockedKeys)->matches($key); + } + }, 20_000); + + expect($held)->toBeLessThan($lookedUp / 2); + })->skip(runningWithCoverage(), 'Timings are meaningless under coverage instrumentation.'); +}); + +describe('Entropy skips what it cannot match', function (): void { + it('is far cheaper for values below min_length', function (): void { + $config = RedactorConfig::fromConfig('default'); + $context = new RedactionContext($config); + $strategy = new ShannonEntropyStrategy; + + $short = ['info', 'GET', '/orders/42', 'Bob', 'pending', 'v2.14.1']; + $long = [str_repeat('Zx7Qm4Kd9Rb2Vn6Tp1Ws8Yc3Hf ', 1)]; + + $shortCost = fastest(function () use ($strategy, $short, $context): void { + foreach ($short as $v) { + if ($strategy->shouldHandle($v, 'k', $context)) { + $strategy->handle($v, 'k', $context); + } + } + $context->discardPendingDetections(); + }, 20_000); + + $longCost = fastest(function () use ($strategy, $long, $context): void { + foreach ($long as $v) { + if ($strategy->shouldHandle($v, 'k', $context)) { + $strategy->handle($v, 'k', $context); + } + } + $context->discardPendingDetections(); + }, 20_000); + + // Six short values must cost less than one value that clears the gate. + expect($shortCost)->toBeLessThan($longCost); + })->skip(runningWithCoverage(), 'Timings are meaningless under coverage instrumentation.'); +}); + +describe('An unchanged payload is not rebuilt', function (): void { + it('costs less to redact a clean payload than a matching one', function (): void { + $redactor = resolve(Redactor::class); + + // Same shape, same size: the only difference is whether anything + // matches, so the gap is the copy that no longer happens. + $clean = ['a' => ['x' => 'plain'], 'b' => ['y' => 'plain'], 'c' => ['z' => 'plain']]; + $dirty = ['a' => ['x' => 'a@b.com'], 'b' => ['y' => 'plain'], 'c' => ['z' => 'plain']]; + + $cleanCost = fastest(fn () => $redactor->redact($clean, 'default'), 10_000); + $dirtyCost = fastest(fn () => $redactor->redact($dirty, 'default'), 10_000); + + expect($cleanCost)->toBeLessThan($dirtyCost); + })->skip(runningWithCoverage(), 'Timings are meaningless under coverage instrumentation.'); + + it('hands back the very same array when nothing matched', function (): void { + $payload = ['a' => ['x' => 'plain'], 'b' => 'also plain']; + + $result = resolve(Redactor::class)->redact($payload, 'default'); + + expect($result)->toBe($payload); + }); +}); diff --git a/tests/Performance/KeyMatcherThroughputTest.php b/tests/Performance/KeyMatcherThroughputTest.php new file mode 100644 index 0000000..b4f7eb6 --- /dev/null +++ b/tests/Performance/KeyMatcherThroughputTest.php @@ -0,0 +1,52 @@ + KeyMatcher::flush()); + + it('is markedly faster than rebuilding a regex per key', function (): void { + $patterns = ['password', '*token*', '*key*', '*secret*', 'authorization', 'user_*_data']; + $keys = ['user_id', 'created_at', 'api_token', 'normal_field', 'trace_id', 'status']; + + $matcher = KeyMatcher::for($patterns); + $iterations = 20_000; + + $start = hrtime(true); + for ($i = 0; $i < $iterations; $i++) { + foreach ($keys as $key) { + $matcher->matches($key); + } + } + $compiled = hrtime(true) - $start; + + // The previous implementation, verbatim, so the regression this guards + // against is the actual one. + $start = hrtime(true); + for ($i = 0; $i < $iterations; $i++) { + foreach ($keys as $key) { + $keyLower = strtolower($key); + foreach ($patterns as $pattern) { + if (! str_contains($pattern, '*')) { + if ($keyLower === strtolower($pattern)) { + break; + } + + continue; + } + $regex = '/^'.str_replace('\*', '.*', preg_quote($pattern, '/')).'$/i'; + if (preg_match($regex, $keyLower) === 1) { + break; + } + } + } + } + $rebuilt = hrtime(true) - $start; + + // Measured at roughly 8x uninstrumented; asserting 2x leaves generous + // room for a loaded runner while still failing on a real regression. + expect($compiled)->toBeLessThan($rebuilt / 2); + }); +})->skip(runningWithCoverage(), 'Timings are meaningless under coverage instrumentation.'); diff --git a/tests/Performance/PathRuleThroughputTest.php b/tests/Performance/PathRuleThroughputTest.php new file mode 100644 index 0000000..55fafb6 --- /dev/null +++ b/tests/Performance/PathRuleThroughputTest.php @@ -0,0 +1,158 @@ + [ + 'method' => 'POST', + 'path' => '/v1/orders', + 'headers' => [ + 'authorization' => 'Bearer eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0NSJ9.abcdefgh', + 'user_agent' => 'Mozilla/5.0', + 'accept' => 'application/json', + 'x_request_id' => 'req_01H8XYZ', + ], + ], + 'user' => ['id' => 42, 'email' => 'alice@customer.com', 'name' => 'Alice'], + 'items' => array_map(fn (int $i): array => [ + 'sku' => "SKU-{$i}", + 'qty' => $i, + 'note' => 'an ordinary line of descriptive text', + ], range(1, 20)), + ]; +} + +/** The best of several short runs: what the code costs, not what the machine was doing. */ +function timeProfile(string $profile, int $iterations = 100, int $runs = 5): float +{ + $redactor = resolve(Redactor::class); + $payload = apiPayload(); + + $redactor->redact($payload, $profile); + + $best = PHP_FLOAT_MAX; + + for ($run = 0; $run < $runs; $run++) { + $start = hrtime(true); + for ($i = 0; $i < $iterations; $i++) { + $redactor->redact($payload, $profile); + } + $best = min($best, (hrtime(true) - $start) / $iterations); + } + + return $best; +} + +function throughputProfile(array $overrides): array +{ + return array_merge([ + 'enabled' => true, + 'strategies' => [BlockedKeysStrategy::class, RegexPatternsStrategy::class, ShannonEntropyStrategy::class], + 'safe_keys' => [], + 'blocked_keys' => ['password', '*token*', '*secret*', '*key*', 'authorization'], + 'patterns' => [ + 'email' => '/[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+/', + 'jwt' => '/eyJ[a-zA-Z0-9_-]*\.eyJ[a-zA-Z0-9_-]*\.[a-zA-Z0-9_-]+/', + 'phone' => '/\b\d{3}[.-]?\d{3}[.-]?\d{4}\b/', + ], + 'paths' => [], + 'operators' => ['default' => 'redact'], + 'replacement' => '[REDACTED]', + 'mark_redacted' => false, + 'track_redacted_keys' => false, + 'non_redactable_object_behavior' => 'preserve', + 'max_value_length' => null, + 'redact_large_objects' => false, + 'max_object_size' => 1000, + 'shannon_entropy' => [ + 'enabled' => true, + 'threshold' => 4.5, + 'min_length' => 20, + 'exclusion_patterns' => [], + ], + ], $overrides); +} + +describe('Path rules as a fast lane', function (): void { + it('is faster than scanning the same payload for the same values', function (): void { + // Same payload, same two things removed. One profile finds them by + // scanning every string; the other is told where they are. + config()->set('redactor.profiles.by_scanning', throughputProfile([])); + + config()->set('redactor.profiles.by_path', throughputProfile([ + // Nothing to scan for: the locations are known. + 'blocked_keys' => [], + 'patterns' => [], + 'shannon_entropy' => ['enabled' => false], + 'paths' => [ + 'request.headers.authorization' => 'redact', + 'user.email' => 'redact', + ], + ])); + + $scanning = timeProfile('by_scanning'); + $paths = timeProfile('by_path'); + + // Both must actually redact the same two values, or the comparison is + // meaningless. + $scanned = resolve(Redactor::class)->redact(apiPayload(), 'by_scanning'); + $pathed = resolve(Redactor::class)->redact(apiPayload(), 'by_path'); + + expect($scanned['request']['headers']['authorization'])->toBe('[REDACTED]') + ->and($pathed['request']['headers']['authorization'])->toBe('[REDACTED]') + ->and($scanned['user']['email'])->toBe('[REDACTED]') + ->and($pathed['user']['email'])->toBe('[REDACTED]') + ->and($paths)->toBeLessThan($scanning); + }); + + it('costs almost nothing when no path rule can match', function (): void { + // An exhausted cursor stops being consulted, so a profile carrying path + // rules that never fire should not pay much for them. + config()->set('redactor.profiles.no_paths', throughputProfile([])); + config()->set('redactor.profiles.dead_paths', throughputProfile([ + 'paths' => [ + 'nothing.here.at.all' => 'redact', + 'also.not.this' => 'redact', + 'or.this.one.either' => 'redact', + ], + ])); + + $without = timeProfile('no_paths'); + $with = timeProfile('dead_paths'); + + expect($with)->toBeLessThan($without * 1.5); + }); + + it('does not slow down as the number of path rules grows', function (): void { + // The trie is walked in lockstep with the payload, so cost tracks the + // rules currently in play - not how many were configured. + $few = ['request.headers.authorization' => 'redact']; + + $many = $few; + for ($i = 0; $i < 200; $i++) { + $many["unused_{$i}.deep.path.{$i}"] = 'redact'; + } + + config()->set('redactor.profiles.few_paths', throughputProfile([ + 'blocked_keys' => [], 'patterns' => [], 'shannon_entropy' => ['enabled' => false], + 'paths' => $few, + ])); + config()->set('redactor.profiles.many_paths', throughputProfile([ + 'blocked_keys' => [], 'patterns' => [], 'shannon_entropy' => ['enabled' => false], + 'paths' => $many, + ])); + + $few = timeProfile('few_paths'); + $many = timeProfile('many_paths'); + + // 200x the rules for well under 2x the time. + expect($many)->toBeLessThan($few * 2.0); + }); +})->skip(runningWithCoverage(), 'Timings are meaningless under coverage instrumentation.'); diff --git a/tests/Performance/RedactionThroughputTest.php b/tests/Performance/RedactionThroughputTest.php new file mode 100644 index 0000000..0fb3d04 --- /dev/null +++ b/tests/Performance/RedactionThroughputTest.php @@ -0,0 +1,173 @@ + 'info', + 'event' => 'request.handled', + 'trace_id' => 'abc123', + 'user' => ['id' => 42, 'email' => 'a@b.com', 'name' => 'Bob', 'password' => 'x'], + 'request' => [ + 'method' => 'GET', + 'path' => '/orders/42', + 'headers' => ['authorization' => 'Bearer zzz', 'user_agent' => 'Mozilla/5.0'], + ], + 'meta' => array_fill_keys(array_map(fn (int $i): string => "field_{$i}", range(1, 20)), 'value-string-here'), + ]; +} + +/** + * Nanoseconds for one redaction of the standard payload. + * + * The best of several short runs, not the mean of one long one: under a + * parallel test run the mean absorbs every context switch on the machine, + * while the fastest block is what the code actually costs. + */ +function timeRedaction(string $profile, int $iterations = 100, int $runs = 7): float +{ + $redactor = resolve(Redactor::class); + $payload = logPayload(); + + $redactor->redact($payload, $profile); // warm the strategy cache + + $best = PHP_FLOAT_MAX; + + for ($run = 0; $run < $runs; $run++) { + $start = hrtime(true); + for ($i = 0; $i < $iterations; $i++) { + $redactor->redact($payload, $profile); + } + $best = min($best, (hrtime(true) - $start) / $iterations); + } + + return $best; +} + +/** Nanoseconds for a trivial loop iteration, to normalise for machine speed. */ +function calibration(int $iterations = 100_000, int $runs = 5): float +{ + $best = PHP_FLOAT_MAX; + + for ($run = 0; $run < $runs; $run++) { + $sink = 0; + + $start = hrtime(true); + for ($i = 0; $i < $iterations; $i++) { + $sink += $i % 7; + } + + $best = min($best, (hrtime(true) - $start) / $iterations); + } + + return $best; +} + +describe('Redaction throughput', function (): void { + it('redacts a realistic log context within budget for its machine', function (): void { + $unit = calibration(); + $perRedaction = timeRedaction('default'); + + // Measured at roughly 5,000 calibration units on PHP 8.5. The ceiling + // is set well above that so an ordinary runner never fails, while a + // change that makes redaction quadratic still does. + expect($perRedaction / $unit)->toBeLessThan(50_000.0); + }); + + it('keeps the performance profile faster than the default', function (): void { + // The performance profile exists to skip work. If it stops being + // faster, it has stopped doing its job. + expect(timeRedaction('performance'))->toBeLessThan(timeRedaction('default')); + }); + + it('keeps the default profile faster than strict', function (): void { + expect(timeRedaction('default'))->toBeLessThan(timeRedaction('strict')); + }); +})->skip(runningWithCoverage(), 'Timings are meaningless under coverage instrumentation.'); + +describe('Redaction scaling', function (): void { + it('scales linearly with payload size, not quadratically', function (): void { + $redactor = resolve(Redactor::class); + + $build = fn (int $n): array => array_fill_keys( + array_map(fn (int $i): string => "field_{$i}", range(1, $n)), + 'some ordinary value' + ); + + $small = $build(2_000); + $large = $build(20_000); + + config()->set('redactor.profiles.scaling', array_merge( + config('redactor.profiles.default'), + ['redact_large_objects' => false, 'mark_redacted' => false] + )); + + $redactor->redact($small, 'scaling'); + + $start = hrtime(true); + $redactor->redact($small, 'scaling'); + $smallTime = hrtime(true) - $start; + + $start = hrtime(true); + $redactor->redact($large, 'scaling'); + $largeTime = hrtime(true) - $start; + + // 10x the input should cost roughly 10x. Quadratic behaviour would be + // 100x; the ceiling of 30x absorbs GC and cache noise. + expect($largeTime / max($smallTime, 1))->toBeLessThan(30.0); + })->skip(runningWithCoverage(), 'Timings are meaningless under coverage instrumentation.'); + + it('holds memory flat for a large payload', function (): void { + config()->set('redactor.profiles.scaling', array_merge( + config('redactor.profiles.default'), + ['redact_large_objects' => false, 'mark_redacted' => false] + )); + + $payload = array_fill_keys( + array_map(fn (int $i): string => "field_{$i}", range(1, 50_000)), + 'value with some text in it' + ); + + $before = memory_get_usage(); + resolve(Redactor::class)->redact($payload, 'scaling'); + $growthMb = (memory_get_usage() - $before) / 1_048_576; + + // The redacted copy is the only allocation that should scale with the + // input; anything holding per-node state would blow past this. + expect($growthMb)->toBeLessThan(64.0); + }); + + it('does not let the entropy cache grow without bound across calls', function (): void { + // The cache lives on RedactionContext, which is per-redaction. A cache + // that outlived a call would grow forever in a long-running worker. + config()->set('redactor.profiles.scaling', array_merge( + config('redactor.profiles.default'), + ['mark_redacted' => false] + )); + + $redactor = resolve(Redactor::class); + + for ($i = 0; $i < 200; $i++) { + $redactor->redact(['note' => "unique-string-number-{$i}-with-padding"], 'scaling'); + } + + $before = memory_get_usage(); + + for ($i = 0; $i < 2_000; $i++) { + $redactor->redact(['note' => "another-unique-string-{$i}-with-padding"], 'scaling'); + } + + expect((memory_get_usage() - $before) / 1_048_576)->toBeLessThan(4.0); + }); +}); diff --git a/tests/Pest.php b/tests/Pest.php index 044de62..29da0f4 100644 --- a/tests/Pest.php +++ b/tests/Pest.php @@ -1,5 +1,7 @@ extend(Tests\TestCase::class) +pest()->extend(TestCase::class) // ->use(Illuminate\Foundation\Testing\RefreshDatabase::class) - ->in('Feature', 'Unit'); + ->in('Feature', 'Unit', 'Performance'); /* |-------------------------------------------------------------------------- @@ -26,9 +28,7 @@ | */ -expect()->extend('toBeOne', function () { - return $this->toBe(1); -}); +expect()->extend('toBeOne', fn () => $this->toBe(1)); /* |-------------------------------------------------------------------------- @@ -41,7 +41,61 @@ | */ -function something() +/** + * A fixed pseudonymization key for tests. + * + * Surrogates are only stable for a given key, so assertions about them need a + * known one. Defined here rather than as a per-file constant so every suite + * agrees, including when a single file is run in isolation. + */ +function testPseudonymizationKey(): string +{ + return 'a-test-pseudonymization-key-of-sufficient-length'; +} + +/** + * Whether a coverage driver is actively instrumenting this run. + * + * Instrumentation dominates the clock and flattens the difference between a + * fast and a slow implementation, so timing assertions made under it are + * meaningless - a benchmark that shows 8x uninstrumented showed 1.4x under + * pcov in CI. Timing-sensitive tests skip themselves rather than flake. + */ +function runningWithCoverage(): bool +{ + if (extension_loaded('pcov') && (bool) ini_get('pcov.enabled')) { + return true; + } + + return extension_loaded('xdebug') + && str_contains((string) ini_get('xdebug.mode'), 'coverage'); +} + +/** + * Clean up directory recursively + */ +function cleanupDirectory(string $dir): void { - // .. + if (! is_dir($dir)) { + return; + } + + $files = scandir($dir); + foreach ($files as $file) { + if ($file === '.' || $file === '..') { + continue; + } + + $path = $dir.'/'.$file; + + if (is_dir($path)) { + cleanupDirectory($path); + } else { + // Ensure file is writable before deletion + chmod($path, 0644); + unlink($path); + } + } + + rmdir($dir); } diff --git a/tests/Unit/DecoderTest.php b/tests/Unit/DecoderTest.php new file mode 100644 index 0000000..e43e607 --- /dev/null +++ b/tests/Unit/DecoderTest.php @@ -0,0 +1,66 @@ +toHaveCount(1) + ->and($derived[0]->encoding)->toBe('json') + ->and($derived[0]->text)->toContain('postgres://app:s3cr3t@db/x') + ->and($derived[0]->offset)->toBe(2) + ->and(substr($window, $derived[0]->offset, $derived[0]->length))->toStartWith(' "db"'); + }); + + it('decodes a base64 token that holds text and skips words and binaries', function (): void { + $secret = base64_encode('STRIPE=sk_live_4eC39HqLyjWDarjtT1zdp7dc'); + $binary = base64_encode(random_bytes(30)); + $window = "data:\n key: {$secret}\n blob: {$binary}\n word: Authorization\n"; + + $derived = Decoder::derive($window); + $base64 = array_values(array_filter($derived, fn (DerivedSubject $d): bool => $d->encoding === 'base64')); + + expect($base64)->toHaveCount(1) + ->and($base64[0]->text)->toBe('STRIPE=sk_live_4eC39HqLyjWDarjtT1zdp7dc') + ->and(substr($window, $base64[0]->offset, $base64[0]->length))->toBe($secret); + }); + + it('decodes a percent-encoded run', function (): void { + $window = 'GET /cb?token=sk_live_4eC39HqLyjWDarjtT1zdp7dc%26redirect%3Dhttps%3A%2F%2Fu%3Ap%40h'; + + $derived = Decoder::derive($window); + $url = array_values(array_filter($derived, fn (DerivedSubject $d): bool => $d->encoding === 'url')); + + expect($url)->not->toBeEmpty() + ->and($url[0]->text)->toContain('https://u:p@h'); + }); + + it('derives nothing from plain text', function (): void { + expect(Decoder::derive("just a line\nand another\n"))->toBe([]); + }); +}); + +describe('Decoder when the engine gives up', function (): void { + it('derives nothing rather than throwing or returning half a result', function (): void { + $window = 'GET /cb?token=%73%6b%5f%6c%69%76%65 '.str_repeat('A', 25)."\n"; + + expect(Decoder::derive($window))->not->toBe([]); + + $limit = (string) ini_get('pcre.backtrack_limit'); + ini_set('pcre.backtrack_limit', '1'); + + try { + // Sanity check: with the limit this low the engine really does give up on anything that backtracks. + expect(@preg_match('/a*ab/', 'aaab'))->toBeFalse() + ->and(Decoder::derive($window))->toBe([]); + } finally { + ini_set('pcre.backtrack_limit', $limit); + } + }); +}); diff --git a/tests/Unit/FileCollectorTest.php b/tests/Unit/FileCollectorTest.php new file mode 100644 index 0000000..ef0684e --- /dev/null +++ b/tests/Unit/FileCollectorTest.php @@ -0,0 +1,238 @@ + $contents) { + $full = $base.'/'.$path; + @mkdir(dirname($full), 0777, true); + file_put_contents($full, $contents); + } + + return $base; +} + +/** @return array relative paths, sorted */ +function collected(string $base, array $patterns = [], int $max = 10_485_760, bool $skipBinary = true, bool $gitignore = true): array +{ + $files = FileCollector::collect([$base], $patterns, $max, $skipBinary, $gitignore); + + $real = realpath($base); + $relative = array_map( + fn (string $f): string => ltrim(str_replace((string) $real, '', $f), '/'), + $files + ); + + sort($relative); + + return $relative; +} + +describe('FileCollector exclusions', function (): void { + it('excludes directories named by a path pattern', function (): void { + // notName() compares basenames only, so the shipped 'vendor/*' and + // 'node_modules/*' defaults could never match and every dependency in + // the project was scanned. + $base = tree([ + 'app.php' => 'ok', + 'vendor/pkg/a.php' => 'secret@leak.com', + 'node_modules/x/b.js' => 'secret@leak.com', + ]); + + expect(collected($base, ['vendor/*', 'node_modules/*']))->toBe(['app.php']); + + cleanupDirectory($base); + }); + + it('excludes nested files under an excluded directory', function (): void { + $base = tree([ + 'keep.php' => 'ok', + 'vendor/a/b/c/deep.php' => 'x', + ]); + + expect(collected($base, ['vendor/*']))->toBe(['keep.php']); + + cleanupDirectory($base); + }); + + it('still excludes by basename glob', function (): void { + $base = tree([ + 'composer.lock' => 'x', + 'app.min.js' => 'x', + 'sub/other.lock' => 'x', + 'keep.php' => 'ok', + ]); + + expect(collected($base, ['*.lock', '*.min.js']))->toBe(['keep.php']); + + cleanupDirectory($base); + }); + + it('collects everything when no patterns are given', function (): void { + $base = tree(['a.php' => 'x', 'sub/b.php' => 'x']); + + expect(collected($base))->toBe(['a.php', 'sub/b.php']); + + cleanupDirectory($base); + }); + + it('ignores an empty pattern rather than excluding everything', function (): void { + $base = tree(['a.php' => 'x']); + + expect(collected($base, ['']))->toBe(['a.php']); + + cleanupDirectory($base); + }); + + it('scans a file named explicitly even when a pattern would exclude it', function (): void { + $base = tree(['vendor/pkg/a.php' => 'x']); + + $files = FileCollector::collect([$base.'/vendor/pkg/a.php'], ['vendor/*']); + + expect($files)->toHaveCount(1); + + cleanupDirectory($base); + }); +}); + +describe('FileCollector eligibility', function (): void { + it('skips files over the size limit', function (): void { + $base = tree([ + 'small.txt' => str_repeat('a', 10), + 'big.txt' => str_repeat('a', 5000), + ]); + + expect(collected($base, [], 1000))->toBe(['small.txt']); + + cleanupDirectory($base); + }); + + it('skips binary files', function (): void { + // Random bytes score high entropy, so every binary in the tree used to + // come back as a finding. + $base = tree([ + 'text.txt' => "hello\nworld\n", + 'image.bin' => "\x89PNG\r\n\x1a\n\x00\x00\x00\x0dIHDR".random_bytes(512), + ]); + + expect(collected($base))->toBe(['text.txt']); + + cleanupDirectory($base); + }); + + it('keeps binary files when skip_binary is off', function (): void { + $base = tree([ + 'text.txt' => 'hello', + 'image.bin' => "\x00\x01\x02\x03", + ]); + + expect(collected($base, [], 10_485_760, false))->toBe(['image.bin', 'text.txt']); + + cleanupDirectory($base); + }); + + it('keeps text in a legacy encoding, which is not binary', function (): void { + $base = tree(['latin1.txt' => "caf\xe9 cr\xe8me br\xfbl\xe9e\n".str_repeat("R\xe9sum\xe9 de la r\xe9union\n", 40)]); + + expect(collected($base))->toHaveCount(1); + }); + + it('skips NUL-free binary by its control bytes', function (): void { + $bytes = ''; + for ($i = 1; $i < 256; $i++) { + $bytes .= chr($i); + } + + $base = tree(['blob.bin' => str_repeat($bytes, 8)]); + + expect(collected($base))->toHaveCount(0); + }); + + it('keeps UTF-8 text that is not ASCII', function (): void { + $base = tree([ + 'japanese.txt' => '日本語のテキストです', + 'accents.txt' => 'café naïve', + ]); + + expect(collected($base))->toBe(['accents.txt', 'japanese.txt']); + + cleanupDirectory($base); + }); + + it('keeps an empty file', function (): void { + $base = tree(['empty.txt' => '']); + + expect(collected($base))->toBe(['empty.txt']); + + cleanupDirectory($base); + }); + + it('skips unreadable files', function (): void { + $base = tree(['secret.txt' => 'x', 'open.txt' => 'y']); + chmod($base.'/secret.txt', 0000); + + expect(collected($base))->toBe(['open.txt']); + + cleanupDirectory($base); + })->skip(posix_geteuid() === 0, 'chmod does not restrict root'); + + it('silently ignores paths that do not exist', function (): void { + expect(FileCollector::collect(['/no/such/path/at/all']))->toBe([]); + }); + + it('deduplicates a file reached by two paths', function (): void { + $base = tree(['a.php' => 'x']); + + expect(FileCollector::collect([$base, $base.'/a.php']))->toHaveCount(1); + + cleanupDirectory($base); + }); +}); + +describe('FileCollector gitignore awareness', function (): void { + it('skips files git is ignoring', function (): void { + $base = tree([ + '.gitignore' => "ignored.txt\n", + 'ignored.txt' => 'x', + 'kept.txt' => 'y', + ]); + + exec('git -C '.escapeshellarg($base).' init -q 2>/dev/null'); + + expect(collected($base))->not->toContain('ignored.txt') + ->and(collected($base))->toContain('kept.txt'); + + cleanupDirectory($base); + }); + + it('includes them when respect_gitignore is off', function (): void { + $base = tree([ + '.gitignore' => "ignored.txt\n", + 'ignored.txt' => 'x', + ]); + + exec('git -C '.escapeshellarg($base).' init -q 2>/dev/null'); + + expect(collected($base, [], 10_485_760, true, false))->toContain('ignored.txt'); + + cleanupDirectory($base); + }); +}); + +describe('FileCollector binary sniffing', function (): void { + it('skips content that is not valid UTF-8 even when it has no NUL byte', function (): void { + $base = tree([ + 'blob.bin' => str_repeat("\x80\x81\x82\x83\x84\x85\x86\x87", 64), + 'text.txt' => 'ok', + ]); + + expect(collected($base))->toBe(['text.txt']); + + cleanupDirectory($base); + }); +}); diff --git a/tests/Unit/PatchParserTest.php b/tests/Unit/PatchParserTest.php new file mode 100644 index 0000000..158b8ff --- /dev/null +++ b/tests/Unit/PatchParserTest.php @@ -0,0 +1,115 @@ + 'AKIAIOSFODNN7EXAMPLE', ++ 'other' => 'x', +@@ -40 +42 @@ +- 'old' => 1, ++ 'new' => 2, +DIFF; + + $patches = PatchParser::parse($diff); + + expect($patches)->toHaveCount(1) + ->and($patches[0]->path)->toBe('config/app.php') + ->and($patches[0]->addedLines)->toBe([ + 11 => " 'key' => 'AKIAIOSFODNN7EXAMPLE',", + 12 => " 'other' => 'x',", + 42 => " 'new' => 2,", + ]) + ->and($patches[0]->lineAt(3))->toBe(42); + }); + + it('splits several files and skips deletions and binaries', function (): void { + $diff = <<<'DIFF' +diff --git a/a.txt b/a.txt +--- a/a.txt ++++ b/a.txt +@@ -0,0 +1 @@ ++alpha +diff --git a/gone.txt b/gone.txt +deleted file mode 100644 +--- a/gone.txt ++++ /dev/null +@@ -1 +0,0 @@ +-bye +diff --git a/logo.png b/logo.png +Binary files a/logo.png and b/logo.png differ +diff --git a/b.txt b/b.txt +--- a/b.txt ++++ b/b.txt +@@ -3,0 +4 @@ ++beta +DIFF; + + $paths = array_map(fn (Patch $p): string => $p->path, PatchParser::parse($diff)); + + expect($paths)->toBe(['a.txt', 'b.txt']); + }); + + it('attaches the commit hash from log output and separates commits', function (): void { + $diff = <<<'DIFF' +commit 0123456789abcdef0123456789abcdef01234567 +diff --git a/x b/x +--- a/x ++++ b/x +@@ -0,0 +1 @@ ++one +commit fedcba9876543210fedcba9876543210fedcba98 +diff --git a/x b/x +--- a/x ++++ b/x +@@ -1,0 +2 @@ ++two +DIFF; + + $patches = PatchParser::parse($diff); + + expect($patches)->toHaveCount(2) + ->and($patches[0]->commit)->toBe('0123456789abcdef0123456789abcdef01234567') + ->and($patches[0]->addedLines)->toBe([1 => 'one']) + ->and($patches[1]->commit)->toBe('fedcba9876543210fedcba9876543210fedcba98') + ->and($patches[1]->addedLines)->toBe([2 => 'two']); + }); + + it('unquotes a path git had to quote and handles context lines', function (): void { + $diff = <<<'DIFF' +diff --git "a/dir/sp ace.txt" "b/dir/sp ace.txt" +--- "a/dir/sp ace.txt" ++++ "b/dir/sp ace.txt" +@@ -1,2 +1,3 @@ + first ++inserted + last +DIFF; + + $patches = PatchParser::parse($diff); + + expect($patches[0]->path)->toBe('dir/sp ace.txt') + ->and($patches[0]->addedLines)->toBe([2 => 'inserted']); + }); + + it('drops a patch that adds nothing', function (): void { + expect(PatchParser::parse("diff --git a/x b/x\n--- a/x\n+++ b/x\n@@ -1 +0,0 @@\n-gone\n"))->toBe([]); + }); + + it('joins the added lines for scanning', function (): void { + $patch = PatchParser::parse("diff --git a/x b/x\n--- a/x\n+++ b/x\n@@ -0,0 +5,2 @@\n+a\n+b\n")[0]; + + expect($patch->text())->toBe("a\nb") + ->and($patch->lineAt(1))->toBe(5) + ->and($patch->lineAt(2))->toBe(6); + }); +}); diff --git a/tests/Unit/ScannerTest.php b/tests/Unit/ScannerTest.php index 2d4e3dc..b8b85d3 100644 --- a/tests/Unit/ScannerTest.php +++ b/tests/Unit/ScannerTest.php @@ -1,20 +1,20 @@ tempDir = sys_get_temp_dir().'/scanner_test_'.uniqid(); mkdir($this->tempDir, 0755, true); }); - afterEach(function () { + afterEach(function (): void { cleanupDirectory($this->tempDir); }); - it('handles unreadable files gracefully when called directly', function () { + it('handles unreadable files gracefully when called directly', function (): void { $redactor = resolve(Redactor::class); $scanner = new Scanner($redactor); @@ -34,7 +34,7 @@ chmod($unreadableFile, 0644); }); - it('handles non-existent files gracefully when called directly', function () { + it('handles non-existent files gracefully when called directly', function (): void { $redactor = resolve(Redactor::class); $scanner = new Scanner($redactor); @@ -48,7 +48,7 @@ expect($result->path)->toBe($nonExistentFile); }); - it('scans readable files successfully when called directly', function () { + it('scans readable files successfully when called directly', function (): void { $redactor = resolve(Redactor::class); $scanner = new Scanner($redactor); @@ -65,11 +65,12 @@ expect($result->profile)->toBe('file_scan'); }); - it('detects full content redaction when sensitive patterns are found', function () { + it('reports a located finding for each sensitive span', function (): void { $redactor = resolve(Redactor::class); $scanner = new Scanner($redactor); - // Create content with sensitive information - this will get fully redacted + // Create content with sensitive information - the address is replaced, + // the surrounding log lines survive. $sensitiveFile = $this->tempDir.'/sensitive.txt'; $content = "This is a log file.\nContact: john@example.com\nEnd of log."; @@ -83,13 +84,16 @@ expect($result->findings)->toHaveCount(1); $finding = $result->findings[0]; - expect($finding['type'])->toBe('full_content_redacted'); - expect($finding['reason'])->toBe('Entire content was redacted'); - expect($finding['original_length'])->toBe(strlen($content)); - expect($finding['profile'])->toBe('file_scan'); + expect($finding->rule)->toBe('email'); + expect($finding->line)->toBe(2); + expect($finding->column)->toBe(10); + expect($finding->excerpt)->toBe('Contact: [REDACTED]'); + expect($finding->excerpt)->not->toContain('john@example.com'); + expect($finding->profile)->toBe('file_scan'); + expect($finding->fingerprint)->toHaveLength(32); }); - it('detects array-based redaction for structured data', function () { + it('detects array-based redaction for structured data', function (): void { $redactor = resolve(Redactor::class); $scanner = new Scanner($redactor); @@ -120,75 +124,84 @@ expect(count($result->findings))->toBeGreaterThan(0); }); - it('handles array-based redaction with _redacted_keys', function () { - // Mock the Redactor to return array with _redacted metadata - $mockRedactor = Mockery::mock(Redactor::class); - $mockRedactor->shouldReceive('redact') - ->once() - ->andReturn([ - 'user' => 'john', - 'email' => '[REDACTED]', - 'api_key' => '[REDACTED]', - '_redacted' => true, - '_redacted_keys' => [ - [ - 'key' => 'email', - 'type' => 'blocked_key', - 'strategy' => 'BlockedKeysStrategy', - ], - [ - 'key' => 'api_key', - 'type' => 'blocked_key', - 'strategy' => 'BlockedKeysStrategy', - ], - ], - ]); - - $scanner = new Scanner($mockRedactor); - - $testFile = $this->tempDir.'/mock_test.json'; - file_put_contents($testFile, '{"user":"john","email":"test@example.com","api_key":"secret123"}'); - - $result = $scanner->scanFile($testFile, 'default'); + it('reports the key alongside a key-based finding in structured data', function (): void { + $scanner = new Scanner(resolve(Redactor::class)); + + $testFile = $this->tempDir.'/keys.json'; + file_put_contents($testFile, "{\n \"user\": \"john\",\n \"password\": \"supersecret123\"\n}"); + + $result = $scanner->scanFile($testFile, 'file_scan'); - expect($result->skipped)->toBeFalse(); - expect($result->error)->toBeNull(); expect($result->hasFindings())->toBeTrue(); - expect($result->findings)->toHaveCount(2); - expect($result->findings[0]['key'])->toBe('email'); - expect($result->findings[0]['type'])->toBe('blocked_key'); - expect($result->findings[1]['key'])->toBe('api_key'); - expect($result->findings[1]['type'])->toBe('blocked_key'); - expect($result->profile)->toBe('default'); + + $rules = array_map(fn (ScanFinding $finding): string => $finding->rule, $result->findings); + expect($rules)->toContain('password_assignment'); + }); + + it('reports paths relative to a base when given one', function (): void { + $scanner = new Scanner(resolve(Redactor::class)); + + $file = $this->tempDir.'/nested/app.env'; + mkdir(dirname($file), 0777, true); + file_put_contents($file, "AWS_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE\n"); + + $result = $scanner->scanFile($file, 'file_scan', $this->tempDir); + + expect($result->findings[0]->path)->toBe('nested/app.env') + // The absolute path stays on the result for the caller that needs it. + ->and($result->path)->toBe($file); + }); + + it('returns no findings for clean content', function (): void { + $scanner = new Scanner(resolve(Redactor::class)); + + $file = $this->tempDir.'/clean.txt'; + file_put_contents($file, "nothing to see here\njust ordinary prose\n"); + + $result = $scanner->scanFile($file, 'file_scan'); + + expect($result->hasFindings())->toBeFalse() + ->and($result->findings)->toBe([]); + }); + + it('numbers lines correctly in a multi-line file', function (): void { + $scanner = new Scanner(resolve(Redactor::class)); + + $file = $this->tempDir.'/multi.env'; + file_put_contents($file, implode("\n", [ + 'FIRST=ok', + 'SECOND=ok', + 'THIRD=ok', + 'AWS_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE', + 'FIFTH=ok', + ])); + + $result = $scanner->scanFile($file, 'file_scan'); + + $aws = array_values(array_filter( + $result->findings, + fn (ScanFinding $finding): bool => $finding->rule === 'aws_access_key' + )); + + expect($aws)->toHaveCount(1) + ->and($aws[0]->line)->toBe(4); }); }); -/** - * Clean up directory recursively - */ -function cleanupDirectory(string $dir): void -{ - if (! is_dir($dir)) { - return; - } - - $files = scandir($dir); - foreach ($files as $file) { - if ($file === '.' || $file === '..') { - continue; - } - - $path = $dir.'/'.$file; - - if (is_dir($path)) { - cleanupDirectory($path); - } else { - // Ensure file is writable before deletion - chmod($path, 0644); - unlink($path); - } - } - - rmdir($dir); -} +describe('Scanner excerpts', function (): void { + it('truncates a long excerpt so a minified line does not flood the report', function (): void { + $dir = sys_get_temp_dir().'/scanner_excerpt_'.uniqid(); + mkdir($dir); + file_put_contents($dir.'/long.txt', 'AWS_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE '.str_repeat('x', 300)."\n"); + + $result = (new Scanner(resolve(Redactor::class)))->scanFile($dir.'/long.txt', 'file_scan'); + + expect($result->findings)->not->toBeEmpty() + ->and($result->findings[0]->excerpt)->toEndWith('...') + ->and(strlen($result->findings[0]->excerpt))->toBe(203) + ->and($result->findings[0]->excerpt)->not->toContain('AKIAIOSFODNN7EXAMPLE'); + + cleanupDirectory($dir); + }); +}); diff --git a/tests/Unit/ValidatorTest.php b/tests/Unit/ValidatorTest.php new file mode 100644 index 0000000..d1e3af9 --- /dev/null +++ b/tests/Unit/ValidatorTest.php @@ -0,0 +1,126 @@ +toBeTrue() + ->and(Validator::nhs('5114026240'))->toBeTrue() + ->and(Validator::nhs('1234567890'))->toBeFalse() + ->and(Validator::nhs('0988416930'))->toBeFalse() + ->and(Validator::nhs('12345'))->toBeFalse(); + }); + + it('checks Dutch BSNs by the eleven-proof', function (): void { + expect(Validator::bsn('283194443'))->toBeTrue() + ->and(Validator::bsn('123456789'))->toBeFalse() + ->and(Validator::bsn('12345678'))->toBeFalse(); + }); + + it('checks German tax identification numbers', function (): void { + expect(Validator::steuerId('72 096 139 541'))->toBeTrue() + ->and(Validator::steuerId('78325422090'))->toBeTrue() + ->and(Validator::steuerId('72096139542'))->toBeFalse() + ->and(Validator::steuerId('02096139541'))->toBeFalse() + ->and(Validator::steuerId('12345678901'))->toBeFalse() + ->and(Validator::steuerId('11223456789'))->toBeFalse() + ->and(Validator::steuerId('11114567890'))->toBeFalse(); + }); + + it('checks French NIRs including Corsican departments', function (): void { + expect(Validator::nir('1 96 04 20 350 020 61'))->toBeTrue() + ->and(Validator::nir('264122A44815913'))->toBeTrue() + ->and(Validator::nir('1 96 04 20 350 020 62'))->toBeFalse() + ->and(Validator::nir('3 96 04 20 350 020 61'))->toBeFalse(); + }); + + it('checks Spanish DNI and NIE letters', function (): void { + expect(Validator::dni('68334472T'))->toBeTrue() + ->and(Validator::dni('68334472-T'))->toBeTrue() + ->and(Validator::dni('x6732518g'))->toBeTrue() + ->and(Validator::dni('12345678A'))->toBeFalse() + ->and(Validator::dni('X6732518A'))->toBeFalse() + ->and(Validator::dni('1234A'))->toBeFalse(); + }); + + it('checks Italian fiscal codes', function (): void { + expect(Validator::codiceFiscale('RSSMRA85T10A562S'))->toBeTrue() + ->and(Validator::codiceFiscale('rssmra85t10a562s'))->toBeTrue() + ->and(Validator::codiceFiscale('RSSMRA85T10A562T'))->toBeFalse() + ->and(Validator::codiceFiscale('RSSMRA85X10A562S'))->toBeFalse(); + }); + + it('checks Belgian national numbers for both centuries', function (): void { + expect(Validator::belgianNationalNumber('54.04.11-613.25'))->toBeTrue() + ->and(Validator::belgianNationalNumber('13.07.16-349.07'))->toBeTrue() + ->and(Validator::belgianNationalNumber('54.04.11-613.26'))->toBeFalse() + ->and(Validator::belgianNationalNumber('54.04.11'))->toBeFalse(); + }); + + it('checks Swedish personal numbers in both lengths and coordination form', function (): void { + expect(Validator::personnummer('600112-7239'))->toBeTrue() + ->and(Validator::personnummer('19600112-7239'))->toBeTrue() + ->and(Validator::personnummer('610485-0869'))->toBeTrue() + ->and(Validator::personnummer('600112-7238'))->toBeFalse() + ->and(Validator::personnummer('1694600000'))->toBeFalse() + ->and(Validator::personnummer('60011'))->toBeFalse(); + }); + + it('checks Norwegian identity numbers with both control digits', function (): void { + expect(Validator::fodselsnummer('25054326869'))->toBeTrue() + ->and(Validator::fodselsnummer('27089492705'))->toBeTrue() + ->and(Validator::fodselsnummer('25054326868'))->toBeFalse() + ->and(Validator::fodselsnummer('25054326879'))->toBeFalse() + ->and(Validator::fodselsnummer('2505432686'))->toBeFalse(); + }); + + it('checks Canadian SINs and Australian TFNs', function (): void { + expect(Validator::sin('965-232-432'))->toBeTrue() + ->and(Validator::sin('123-456-789'))->toBeFalse() + ->and(Validator::sin('065-232-432'))->toBeFalse() + ->and(Validator::sin('12345678'))->toBeFalse() + ->and(Validator::tfn('261 158 631'))->toBeTrue() + ->and(Validator::tfn('75817404'))->toBeTrue() + ->and(Validator::tfn('123 456 789'))->toBeFalse() + ->and(Validator::tfn('1234567'))->toBeFalse(); + }); +}); + +describe('VAT validation', function (): void { + it('verifies the checksums it knows', function (): void { + foreach (['DE869428760', 'DE246246459', 'NL514465888B07', 'GB436083107', 'GB729304771', 'GB198679577001', 'IT55679497721', 'FR50786240626', 'FRAB123456789', 'BE0190125740', 'SE213467230801'] as $valid) { + expect(Validator::vat($valid))->toBeTrue($valid); + } + + foreach (['DE000000000', 'NL123456789B01', 'GB123456789', 'IT00000000001', 'FR00786240626', 'BE0190125741', 'SE213467230802', 'SE213467230901'] as $invalid) { + expect(Validator::vat($invalid))->toBeFalse($invalid); + } + }); + + it('accepts the rest of the union on format', function (): void { + foreach (['ES B12345674', 'ATU12345678', 'DK12345678', 'FI12345678', 'IE1234567FA', 'PL1234567890', 'PT123456789', 'LU12345678', 'CZ12345678', 'HU12345678', 'RO12', 'SK1234567890', 'SI12345678', 'HR12345678901', 'BG123456789', 'EE123456789', 'LT123456789012', 'LV12345678901', 'CY12345678A', 'MT12345678', 'EL123456789'] as $valid) { + expect(Validator::vat($valid))->toBeTrue($valid); + } + + expect(Validator::vat('ES12345'))->toBeFalse() + ->and(Validator::vat('CY12345678'))->toBeFalse() + ->and(Validator::vat('XX12345678'))->toBeFalse() + ->and(Validator::vat('12345678'))->toBeFalse(); + }); +}); + +describe('Validator registry', function (): void { + it('knows its names and accepts custom validators', function (): void { + expect(Validator::exists('luhn'))->toBeTrue() + ->and(Validator::exists('nope'))->toBeFalse() + ->and(Validator::passes('nope', 'x'))->toBeTrue(); + + Validator::extend('even_length', fn (string $v): bool => strlen($v) % 2 === 0); + + expect(Validator::exists('even_length'))->toBeTrue() + ->and(Validator::passes('even_length', 'ab'))->toBeTrue() + ->and(Validator::passes('even_length', 'abc'))->toBeFalse(); + }); +}); diff --git a/workbench/app/Models/User.php b/workbench/app/Models/User.php index 56d6817..2147425 100644 --- a/workbench/app/Models/User.php +++ b/workbench/app/Models/User.php @@ -6,10 +6,11 @@ use Illuminate\Database\Eloquent\Factories\HasFactory; use Illuminate\Foundation\Auth\User as Authenticatable; use Illuminate\Notifications\Notifiable; +use Workbench\Database\Factories\UserFactory; class User extends Authenticatable { - /** @use HasFactory<\Workbench\Database\Factories\UserFactory> */ + /** @use HasFactory */ use HasFactory, Notifiable; /** diff --git a/workbench/database/factories/UserFactory.php b/workbench/database/factories/UserFactory.php index 83bcdce..775ac6e 100644 --- a/workbench/database/factories/UserFactory.php +++ b/workbench/database/factories/UserFactory.php @@ -10,7 +10,7 @@ /** * @template TModel of \Workbench\Redactor\Models\User * - * @extends \Illuminate\Database\Eloquent\Factories\Factory + * @extends Factory */ class UserFactory extends Factory {