From ff6033457c586bdc9a916a0e9334d007fedcd17b Mon Sep 17 00:00:00 2001 From: David Gageot Date: Fri, 20 Nov 2026 17:26:12 +0100 Subject: [PATCH 01/14] fix: redact the whole Slack token, not just its first segment The slack-access-token rule stopped at the first dash. Slack tokens are dash-separated, with the secret in the last segment, so Redact turned `xoxb---` into `[REDACTED]--`. Match the numeric IDs and the secret segment that follows them, and keep the single-segment shape as a fallback. The match ends with the secret, so text hyphenated onto a token is left alone, and carries on only over another token glued straight onto it. --- portcullis_test.go | 74 ++++++++++++++++++++++++++++++++++++++++++++++ rules.go | 22 ++++++++++++-- 2 files changed, 94 insertions(+), 2 deletions(-) diff --git a/portcullis_test.go b/portcullis_test.go index 5561a82..feaf1f3 100644 --- a/portcullis_test.go +++ b/portcullis_test.go @@ -438,6 +438,80 @@ func TestRedactDoesNotSwallowAdjacentTextAfterSlackRotatingToken(t *testing.T) { "adjacent hostname must NOT be swallowed by the rotating-token regex: %q", out) } +// TestRedactCoversWholeSlackAccessToken pins the span of the +// slack-access-token rule. Slack tokens are dash-separated and the +// secret is the last segment (`xoxb---`), so +// a match that stops at the first dash leaves the secret in the +// output. +func TestRedactCoversWholeSlackAccessToken(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + token string + }{ + {"bot", "xoxb-" + strings.Repeat("1", 11) + "-" + strings.Repeat("2", 13) + "-" + strings.Repeat("a", 24)}, + {"user", "xoxp-" + strings.Repeat("1", 11) + "-" + strings.Repeat("2", 13) + "-" + strings.Repeat("3", 13) + "-" + strings.Repeat("f", 32)}, + {"session", "xoxs-" + strings.Repeat("1", 11) + "-" + strings.Repeat("2", 11) + "-" + strings.Repeat("3", 11) + "-" + strings.Repeat("f", 64)}, + {"single_segment", "xoxa-" + strings.Repeat("a", 24)}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + in := "SLACK_TOKEN=" + tc.token + " and the rest of the line" + + assert.Equal(t, "SLACK_TOKEN="+portcullis.Marker+" and the rest of the line", portcullis.Redact(in)) + matches := portcullis.Find(in) + require.Len(t, matches, 1) + assert.Equal(t, tc.token, matches[0].Value) + }) + } + + // The first segment still has to look like an ID, so the + // placeholder found in every Slack tutorial keeps passing through. + assert.False(t, portcullis.Contains("xoxb-your-token-here")) +} + +// TestSlackAccessTokenStopsAtItsLastSegment: the match must not run +// on into whatever is hyphenated onto a token, least of all into the +// prefix of a second token, which would then go undetected. +func TestSlackAccessTokenStopsAtItsLastSegment(t *testing.T) { + t.Parallel() + + user := "xoxp-" + strings.Repeat("1", 11) + "-" + strings.Repeat("2", 13) + "-" + strings.Repeat("3", 13) + "-" + strings.Repeat("f", 32) + bot := "xoxb-" + strings.Repeat("1", 11) + "-" + strings.Repeat("2", 13) + "-" + strings.Repeat("a", 24) + legacy := "xoxa-" + strings.Repeat("a", 24) + + assert.Equal(t, portcullis.Marker+"-retry-count", portcullis.Redact(user+"-retry-count")) + assert.Equal(t, portcullis.Marker+"-retry-count", portcullis.Redact(bot+"-retry-count")) + + for _, second := range []string{user, bot, legacy} { + matches := portcullis.Find(user + "-" + second) + require.Len(t, matches, 2) + assert.Equal(t, user, matches[0].Value) + assert.Equal(t, second, matches[1].Value) + } +} + +// TestRedactCoversGluedSlackAccessTokens: with no separator at all, +// the last segment of a token takes the `xoxb` of the next one. What +// is left of that one must not stay behind in clear. +func TestRedactCoversGluedSlackAccessTokens(t *testing.T) { + t.Parallel() + + user := "xoxp-" + strings.Repeat("1", 11) + "-" + strings.Repeat("2", 13) + "-" + strings.Repeat("3", 13) + "-" + strings.Repeat("f", 32) + bot := "xoxb-" + strings.Repeat("1", 11) + "-" + strings.Repeat("2", 13) + "-" + strings.Repeat("a", 24) + legacy := "xoxa-" + strings.Repeat("b", 24) + + for _, glued := range []string{bot + legacy, legacy + bot, user + bot + legacy, legacy + legacy} { + assert.Equal(t, "token="+portcullis.Marker+" end", portcullis.Redact("token="+glued+" end"), glued) + matches := portcullis.Find(glued) + require.Len(t, matches, 1, glued) + assert.Equal(t, glued, matches[0].Value) + } +} + // TestRedactDoesNotSwallowAdjacentTextAfterPrefixedTokens pins the // upper bounds of the rules whose bodies share a character class // with arbitrary trailing text. Without an explicit ceiling on the diff --git a/rules.go b/rules.go index 413f473..f2967bd 100644 --- a/rules.go +++ b/rules.go @@ -88,6 +88,18 @@ func contextual(vendor, body string) string { `(?P` + body + `)(?:[\x60'"\s;]|\\[nr]|$)` } +// chain matches head then body, and carries on over every further +// link and body glued to them without a separator. A body's character +// class also takes the first characters of a token that follows it +// directly (the `sk` of `sk_live_…`), so a match that ended with the +// first body would leave the rest of the second token unmatched and +// in clear. A run of glued tokens comes out as a single match. It +// ends early at a token too short to match without those borrowed +// characters, which nothing short of lookahead can tell apart. +func chain(head, link, body string) string { + return head + `(?:` + body + link + `)*` + body +} + // rules is the source-form catalogue, kept verbatim from upstream so // future updates apply cleanly. [compiledRuleSet] resolves it into // the regex-compiled form actually used at scan time. @@ -177,8 +189,14 @@ func rules() []rule { keywords: []string{"shpss_", "shpat_", "shpca_", "shppa_"}, }, { - // slack-access-token - expression: asSecretGroup(`?Pxox[baprs]-([0-9a-zA-Z]{10,48})`), + // slack-access-token. Slack tokens are dash-separated: one to + // three numeric IDs, then the secret (`xoxb---`, `xoxp----`). + // The match has to reach that last segment, or the redaction + // leaves the secret itself in clear. It stops there too, so + // whatever is hyphenated onto a token stays out of it. The + // second alternative is the original single-segment shape. + expression: asSecretGroup(`?P` + chain(`xox[baprs]-`, `xox[baprs]-`, `(?:(?:[0-9]{8,14}-){1,3}[0-9a-zA-Z]{18,128}|[0-9a-zA-Z]{10,48})`)), keywords: []string{"xoxb-", "xoxa-", "xoxp-", "xoxr-", "xoxs-"}, }, { From e5c2a9a10d9ea303399afbff85d4bfaeab7ff5de Mon Sep 17 00:00:00 2001 From: David Gageot Date: Fri, 20 Nov 2026 17:26:53 +0100 Subject: [PATCH 02/14] fix: redact full-length Stripe secret and restricted keys The sk_/rk_ rules capped the key body at 32 characters, but current Stripe keys carry 99, so Redact left the last 67 characters of the key in the output and Find reported a truncated value. Raise the ceiling to 247, as TruffleHog does. A body also takes the `sk` of a key glued straight onto it, as the old one already did for 24-character keys, leaving the rest of that key in clear. Carry the match on over it. --- portcullis_test.go | 53 ++++++++++++++++++++++++++++++++++++++++++++++ rules.go | 14 ++++++++++-- 2 files changed, 65 insertions(+), 2 deletions(-) diff --git a/portcullis_test.go b/portcullis_test.go index feaf1f3..7f7664c 100644 --- a/portcullis_test.go +++ b/portcullis_test.go @@ -512,6 +512,59 @@ func TestRedactCoversGluedSlackAccessTokens(t *testing.T) { } } +// TestRedactCoversWholeStripeKey pins the body length of the Stripe +// secret and restricted key rules. Current keys (`sk_live_51…`) carry +// a 99-character body, so a lower cap leaves the tail of the key in +// the output. +func TestRedactCoversWholeStripeKey(t *testing.T) { + t.Parallel() + + for _, prefix := range []string{"sk_live_", "sk_test_", "sk_prod_", "rk_live_", "rk_test_", "rk_prod_"} { + t.Run(prefix, func(t *testing.T) { + t.Parallel() + + key := prefix + "51" + strings.Repeat("aB3", 32) + "x" + in := "STRIPE_KEY=" + key + " and the rest of the line" + + assert.Equal(t, "STRIPE_KEY="+portcullis.Marker+" and the rest of the line", portcullis.Redact(in)) + matches := portcullis.Find(in) + require.Len(t, matches, 1) + assert.Equal(t, key, matches[0].Value) + }) + } +} + +// TestRedactCoversGluedStripeKeys: with no separator between two +// keys, the body of the first takes the `sk` / `rk` of the second. +// What is left of the second key must not stay behind in clear. +func TestRedactCoversGluedStripeKeys(t *testing.T) { + t.Parallel() + + for _, prefixes := range [][2]string{ + {"sk_live_", "sk_live_"}, + {"sk_live_", "rk_test_"}, + {"rk_prod_", "sk_test_"}, + } { + // 247 is the body ceiling: just under it, the first body has + // room for only part of the next prefix. + for _, bodyLen := range []int{24, 32, 99, 245, 246, 247} { + glued := prefixes[0] + strings.Repeat("a", bodyLen) + + prefixes[1] + strings.Repeat("b", bodyLen) + + prefixes[0] + strings.Repeat("c", bodyLen) + + assert.Equal(t, "key="+portcullis.Marker+" end", portcullis.Redact("key="+glued+" end"), glued) + matches := portcullis.Find(glued) + require.Len(t, matches, 1, glued) + assert.Equal(t, glued, matches[0].Value) + } + } + + // Only another key carries the match on. An environment-looking + // suffix is not one. + key := "sk_live_" + strings.Repeat("a", 24) + assert.Equal(t, portcullis.Marker+"_prod_environment", portcullis.Redact(key+"_prod_environment")) +} + // TestRedactDoesNotSwallowAdjacentTextAfterPrefixedTokens pins the // upper bounds of the rules whose bodies share a character class // with arbitrary trailing text. Without an explicit ceiling on the diff --git a/rules.go b/rules.go index f2967bd..8554568 100644 --- a/rules.go +++ b/rules.go @@ -100,6 +100,16 @@ func chain(head, link, body string) string { return head + `(?:` + body + link + `)*` + body } +// stripeKey builds the expression of the Stripe secret (`sk`) and +// restricted (`rk`) key rules: `__`. Current keys +// (`sk_live_51…`) carry a 99-character body where the original format +// had 24, and a ceiling that is too low leaves the tail of a key in +// clear, so it sits at TruffleHog's 247. +func stripeKey(prefix string) string { + const env = `_(?:test|live|prod)_` + return `(?i)` + chain(prefix+env, `[sr]k`+env, `[0-9a-z]{10,247}`) +} + // rules is the source-form catalogue, kept verbatim from upstream so // future updates apply cleanly. [compiledRuleSet] resolves it into // the regex-compiled form actually used at scan time. @@ -206,7 +216,7 @@ func rules() []rule { // intentionally NOT redacted: Stripe publishable keys are // designed to be embedded in client-side code and are not // considered secret. - expression: asSecretGroup(`?P(?i)sk_(test|live|prod)_[0-9a-z]{10,32}`), + expression: asSecretGroup(`?P` + stripeKey(`sk`)), keywords: []string{"sk_test_", "sk_live_", "sk_prod_"}, }, { @@ -693,7 +703,7 @@ func rules() []rule { // a restricted key still grants the scoped Stripe permissions // it was issued with, so it must be redacted. The `prod` env // tag was added in 2024 alongside the `test` / `live` modes. - expression: `(?i)rk_(test|live|prod)_[0-9a-z]{10,32}`, + expression: stripeKey(`rk`), keywords: []string{"rk_test_", "rk_live_", "rk_prod_"}, }, { From 850d877847b876d06d0096ff5f5c4197b35c3da3 Mon Sep 17 00:00:00 2001 From: David Gageot Date: Fri, 20 Nov 2026 17:05:19 +0100 Subject: [PATCH 03/14] fix: detect Discord bot tokens for every snowflake prefix A bot token starts with the base64 of the bot's numeric ID, so the ID's leading digit fixes the first two characters. The keyword list had six of the nine possibilities: tokens starting with Mj, Nj or OT (IDs starting with 2, 6 or 9) never reached the regex. The regex took any second character, relying on the keywords to be found somewhere in the input. Make it require one of the nine prefixes itself, so the longer keyword list can't turn more dotted identifiers into tokens. --- aho.go | 2 +- edge_cases_test.go | 20 +++++++++++++++++--- rules.go | 17 +++++++++-------- 3 files changed, 27 insertions(+), 12 deletions(-) diff --git a/aho.go b/aho.go index 3d271c2..6c4acd8 100644 --- a/aho.go +++ b/aho.go @@ -4,7 +4,7 @@ package portcullis // which patterns occurred in a scanned input and which keywords each // rule subscribes to. Storing it as a small array keeps every test // branch-free; the cap of len(kwMask)*64 indices (currently 320) -// accommodates today's catalogue of ~285 unique keywords with limited +// accommodates today's catalogue of ~295 unique keywords with limited // remaining headroom for future rules (an overflow trips a // deterministic panic in [buildAhoCorasick]). type kwMask [5]uint64 diff --git a/edge_cases_test.go b/edge_cases_test.go index b1fcdab..cf5943f 100644 --- a/edge_cases_test.go +++ b/edge_cases_test.go @@ -65,12 +65,14 @@ func TestNoisyKeywordsFalsePositives(t *testing.T) { } } -// TestDiscordTokenPrefixes verifies all documented Discord bot token prefixes. +// TestDiscordTokenPrefixes verifies every prefix a Discord bot token +// can start with. The first segment is the base64 of the bot's numeric +// ID, so the leading decimal digit (1-9) fixes the first two +// characters. func TestDiscordTokenPrefixes(t *testing.T) { t.Parallel() - // Test all documented prefixes - prefixes := []string{"MT", "Mz", "ND", "NT", "Nz", "OD"} + prefixes := []string{"MT", "Mj", "Mz", "ND", "NT", "Nj", "Nz", "OD", "OT"} for _, prefix := range prefixes { t.Run("prefix_"+prefix, func(t *testing.T) { @@ -82,6 +84,18 @@ func TestDiscordTokenPrefixes(t *testing.T) { } } +// TestDiscordTokenNeedsItsPrefix: a prefix elsewhere in the input gets +// the rule past the keyword filter, but must not make a token out of +// a dotted identifier that doesn't start with one itself. +func TestDiscordTokenNeedsItsPrefix(t *testing.T) { + t.Parallel() + + in := "NOTE M" + strings.Repeat("a", 23) + "." + strings.Repeat("B", 6) + "." + strings.Repeat("C", 30) + + assert.False(t, portcullis.Contains(in)) + assert.Equal(t, in, portcullis.Redact(in)) +} + // TestConnectionStringContextPreservation verifies that MongoDB, Postgres, // and Azure Storage connection strings only redact the password/key portion. func TestConnectionStringContextPreservation(t *testing.T) { diff --git a/rules.go b/rules.go index 8554568..0a2d559 100644 --- a/rules.go +++ b/rules.go @@ -776,20 +776,21 @@ func rules() []rule { // discord-bot-token. Three-part dotted format issued for bot // applications: `.<6-char timestamp>.<27+ // char HMAC>`. The first segment is the bot's user-ID base64 - // encoded; current Discord IDs (2018 onwards) base64 to a - // leading `MT`/`Mz`/`ND`/`NT`/`Nz`/`OD` byte pair, which we - // list as keywords so the AC pre-filter still skips inputs - // without a plausible token prefix. The structural shape - // (M-or-N-or-O-prefixed body, two literal dots, fixed segment - // widths) keeps the regex itself specific. + // encoded, so the ID's leading decimal digit (1-9) fixes its + // first two characters: `MT`/`Mj`/`Mz`/`ND`/`NT`/`Nj`/`Nz`/ + // `OD`/`OT`. The regex requires one of the nine, and we list + // them as keywords so the AC pre-filter still skips inputs + // without a plausible token prefix. Together with the two + // literal dots and the fixed segment widths, that keeps the + // rule specific. // // caseSensitive is critical here: the lower-cased forms of // these two-letter prefixes (`mt`, `nd`, `od`, …) are common // English bigrams and trip the keyword pre-filter on ~30% of // arbitrary text files. Pinning the case lets the AC pass // drop the regex run on those files. - expression: `[MNO][A-Za-z\d_-]{23,25}\.[\w-]{6,7}\.[\w-]{27,38}`, - keywords: []string{"MT", "Mz", "ND", "NT", "Nz", "OD"}, + expression: `(?:M[Tjz]|N[DTjz]|O[DT])[A-Za-z\d_-]{22,24}\.[\w-]{6,7}\.[\w-]{27,38}`, + keywords: []string{"MT", "Mj", "Mz", "ND", "NT", "Nj", "Nz", "OD", "OT"}, caseSensitive: true, }, { From 26cf3fbf91c1f6611b36a360109ae7e3457aa238 Mon Sep 17 00:00:00 2001 From: David Gageot Date: Fri, 20 Nov 2026 17:05:53 +0100 Subject: [PATCH 04/14] fix: never report the redaction marker as a secret The password class of the connection-string rules is broad enough to match "[REDACTED]", so Contains(Redact(x)) was true and Find returned the marker for every redacted mongodb / postgres / mysql / redis / amqp URL. Skip spans that are exactly the marker. Contains keeps its allocation-free MatchString path on marker-free input; otherwise it looks at one match more than there are markers. --- portcullis.go | 23 ++++++++++++++++++----- portcullis_test.go | 32 ++++++++++++++++++++++++++++++++ rules.go | 8 ++++++++ 3 files changed, 58 insertions(+), 5 deletions(-) diff --git a/portcullis.go b/portcullis.go index 0e0761b..e7ad19d 100644 --- a/portcullis.go +++ b/portcullis.go @@ -9,7 +9,9 @@ import ( // Marker replaces every detected secret span. Chosen so it doesn't // match any rule's keyword pre-filter — see TestMarkerIsNotASecret -// for the safety property that makes [Redact] idempotent. +// for the safety property that makes [Redact] idempotent. A marker +// sitting where a rule expects a secret (the password of an +// already-redacted connection string, say) is never reported as one. const Marker = "[REDACTED]" // Match describes a single secret span detected in an input. Start @@ -72,7 +74,7 @@ func findMatches(text string) []Match { continue } value := text[s:e] - if compiled.validator != nil && !compiled.validator(value) { + if !compiled.accepts(value) { continue } matches = append(matches, Match{Start: s, End: e, Value: value}) @@ -130,21 +132,32 @@ func Contains(text string) bool { if !ok { return false } + // MatchString can't tell a secret from the marker a previous + // Redact left in its place, so that shortcut is reserved for + // marker-free input. + markers := strings.Count(text, Marker) for i := range rs.rules { r := &rs.rules[i] if !r.passes(found, text) { continue } compiled := r.compile() - if compiled.validator == nil { + if compiled.validator == nil && markers == 0 { if compiled.re.MatchString(text) { return true } continue } - for _, m := range compiled.re.FindAllStringSubmatchIndex(text, -1) { + // A validator can reject any number of matches. Without one + // only markers are rejected, one match per marker at most, so + // the match after those settles it. + limit := -1 + if compiled.validator == nil { + limit = markers + 1 + } + for _, m := range compiled.re.FindAllStringSubmatchIndex(text, limit) { s, e := redactSpan(m, compiled.secretIdx) - if compiled.validator(text[s:e]) { + if compiled.accepts(text[s:e]) { return true } } diff --git a/portcullis_test.go b/portcullis_test.go index 7f7664c..62bfc43 100644 --- a/portcullis_test.go +++ b/portcullis_test.go @@ -277,6 +277,8 @@ func TestContainsRecognisesKnownTokens(t *testing.T) { assert.NotContainsf(t, out, tc.text, "raw secret must be gone after Redact: %q", out) assert.Containsf(t, out, portcullis.Marker, "redaction marker must appear in %q", out) + assert.Falsef(t, portcullis.Contains(out), + "redacted output must no longer trip Contains: %q", out) }) } } @@ -733,6 +735,36 @@ func TestMarkerIsNotASecret(t *testing.T) { portcullis.Redact("prefix "+portcullis.Marker+" suffix")) } +// TestMarkerInSecretPositionIsNotASecret covers the rules whose secret +// class is broad enough to match the marker itself, such as the +// password of a connection string. Reporting it would make every +// already-redacted connection string look like a leak. +func TestMarkerInSecretPositionIsNotASecret(t *testing.T) { + t.Parallel() + + redacted := "redis://:" + portcullis.Marker + "@cache.example.com:6379" + + assert.False(t, portcullis.Contains(redacted)) + assert.Empty(t, portcullis.Find(redacted)) + assert.Equal(t, redacted, portcullis.Redact(redacted)) + + // A real secret is still reported: one that another rule finds + // next to the marker, one that the same rule finds after it, and + // one wrapped around it. + pat := "dckr_pat_" + strings.Repeat("A", 27) + matches := portcullis.Find(redacted + " " + pat) + require.Len(t, matches, 1) + assert.Equal(t, pat, matches[0].Value) + assert.True(t, portcullis.Contains(redacted+" "+pat)) + + matches = portcullis.Find(redacted + " " + redacted + " redis://:hunter2hunter2@replica.example.com:6379") + require.Len(t, matches, 1) + assert.Equal(t, "hunter2hunter2", matches[0].Value) + assert.True(t, portcullis.Contains(redacted+" "+redacted+" redis://:hunter2hunter2@replica.example.com:6379")) + + assert.True(t, portcullis.Contains("redis://:hunter2"+portcullis.Marker+"@cache.example.com:6379")) +} + // TestRedactDetectsSecretsAcrossWordBoundaries exercises the change // that dropped the leading and trailing word-boundary anchors from // the rule expressions. Before the change, only secrets that stood diff --git a/rules.go b/rules.go index 0a2d559..4087420 100644 --- a/rules.go +++ b/rules.go @@ -2041,6 +2041,14 @@ type compiledMatch struct { validator func(string) bool } +// accepts reports whether value, a span matched by the rule's regex, +// is a secret: it has to pass the rule's validator, and it must not +// be the [Marker] itself, which broad classes such as a +// connection-string password match in already-redacted text. +func (c *compiledMatch) accepts(value string) bool { + return value != Marker && (c.validator == nil || c.validator(value)) +} + type compiledRule struct { kwBits kwMask caseSensitive bool From 984d5cdbe829efd142b91af59621992d20f66763 Mon Sep 17 00:00:00 2001 From: David Gageot Date: Fri, 20 Nov 2026 17:07:38 +0100 Subject: [PATCH 05/14] fix(scan): scan pipes and FIFOs instead of reporting them clean The reader trusted the stat size, which is 0 for a pipe, so `portcullis-scan <(git diff)` read nothing and exited 0 whatever the content. Read to EOF instead, and enforce -max-size on what was actually read. The stat size still sizes the buffer for small files. --- cmd/portcullis-scan/main.go | 53 ++++++++++------ cmd/portcullis-scan/stream_unix_test.go | 82 +++++++++++++++++++++++++ 2 files changed, 117 insertions(+), 18 deletions(-) create mode 100644 cmd/portcullis-scan/stream_unix_test.go diff --git a/cmd/portcullis-scan/main.go b/cmd/portcullis-scan/main.go index e774fe1..249a543 100644 --- a/cmd/portcullis-scan/main.go +++ b/cmd/portcullis-scan/main.go @@ -41,10 +41,12 @@ package main import ( "bytes" + "errors" "flag" "fmt" "io" "io/fs" + "math" "os" "path/filepath" "runtime" @@ -361,38 +363,53 @@ func readIfScannable(path string, maxSize int64, scanBinary bool) (data []byte, if err != nil { return nil, false, err } - size := st.Size() - if maxSize > 0 && size > maxSize { + if maxSize > 0 && st.Size() > maxSize { return nil, false, nil } - if scanBinary { - data, err = io.ReadAll(f) - if err != nil { - return nil, false, err - } - return data, true, nil + // That size is only a hint: a pipe (`<(git diff)`, a FIFO) reports + // 0 and a file being written to can outgrow it. So read to EOF + // rather than to that size, stopping one byte past maxSize so that + // content over the limit is still skipped. + var r io.Reader = f + if maxSize > 0 && maxSize < math.MaxInt64 { + r = io.LimitReader(f, maxSize+1) } // Sniff first 8 KiB for a NUL byte before committing to read // the rest. On large binaries (git packs, archives, prebuilt // executables) this avoids both the IO and the regex pass. - sniffN := min(int64(binarySniffSize), size) - sniff := make([]byte, sniffN) - if _, err = io.ReadFull(f, sniff); err != nil { + // + // The buffer starts one byte past the stat size, enough to see a + // small file end where it should, and only widens to the full + // window if there turns out to be more. + data = make([]byte, min(st.Size(), binarySniffSize-1)+1) + n, err := io.ReadFull(r, data) + if err == nil && n < binarySniffSize { + data = append(data, make([]byte, binarySniffSize-n)...) + var more int + more, err = io.ReadFull(r, data[n:]) + n += more + } + atEOF := errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) + if err != nil && !atEOF { return nil, false, err } - if bytes.IndexByte(sniff, 0) >= 0 { + data = data[:n] + if !scanBinary && bytes.IndexByte(data, 0) >= 0 { return nil, false, nil } - if size == sniffN { - return sniff, true, nil + if !atEOF { + rest, err := io.ReadAll(r) + if err != nil { + return nil, false, err + } + data = append(data, rest...) } - rest, err := io.ReadAll(f) - if err != nil { - return nil, false, err + if maxSize > 0 && int64(len(data)) > maxSize { + return nil, false, nil } - return append(sniff, rest...), true, nil + return data, true, nil } var valueSanitizer = strings.NewReplacer("\r\n", " ", "\n", " ", "\r", " ") diff --git a/cmd/portcullis-scan/stream_unix_test.go b/cmd/portcullis-scan/stream_unix_test.go new file mode 100644 index 0000000..d7645c2 --- /dev/null +++ b/cmd/portcullis-scan/stream_unix_test.go @@ -0,0 +1,82 @@ +//go:build unix && !aix && !solaris + +package main + +import ( + "bytes" + "os" + "path/filepath" + "strconv" + "syscall" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// newFIFO returns the path of a FIFO that yields content to its first +// reader. A FIFO stands in for any target whose size stat can't know +// up front, such as the pipe behind `portcullis-scan <(git diff)`. +func newFIFO(t *testing.T, content string) string { + t.Helper() + + fifo := filepath.Join(t.TempDir(), "stream") + require.NoError(t, syscall.Mkfifo(fifo, 0o600)) + written := make(chan error, 1) + go func() { + // Blocks until the scanner opens the FIFO. + written <- os.WriteFile(fifo, []byte(content), 0o600) + }() + t.Cleanup(func() { + select { + case err := <-written: + assert.NoError(t, err) + case <-time.After(10 * time.Second): + t.Error("the scanner never opened the FIFO") + } + }) + return fifo +} + +// TestRunScansStreamTarget: stat reports a pipe as 0 bytes long, which +// must not be taken to mean there is nothing to scan. -max-size applies +// to what the pipe actually delivers. +func TestRunScansStreamTarget(t *testing.T) { + t.Parallel() + + const content = "TOKEN=" + githubPAT + "\n" + atLimit := "-max-size=" + strconv.Itoa(len(content)) + + for _, tc := range []struct { + name string + flags []string + content string + found bool + }{ + {"default", nil, content, true}, + {"binary", []string{"-binary"}, content, true}, + {"at max size", []string{atLimit}, content, true}, + {"at max size, binary", []string{atLimit, "-binary"}, content, true}, + {"above max size", []string{atLimit}, content + "\n", false}, + {"above max size, binary", []string{atLimit, "-binary"}, content + "\n", false}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + fifo := newFIFO(t, tc.content) + + var stdout, stderr bytes.Buffer + code := run(append(tc.flags, fifo), &stdout, &stderr) + + assert.Empty(t, stderr.String()) + if tc.found { + assert.Equal(t, exitFound, code) + assert.Equal(t, fifo+":1:7: "+githubPAT+"\n", stdout.String()) + } else { + assert.Equal(t, exitClean, code) + assert.Empty(t, stdout.String()) + } + }) + } +} From 59f8a89158c9b0d75d41871c0e082f36a51e5873 Mon Sep 17 00:00:00 2001 From: David Gageot Date: Fri, 20 Nov 2026 17:07:58 +0100 Subject: [PATCH 06/14] fix(scan): anchor -ignore patterns that start with a slash The leading "/" was stripped before deciding whether a pattern is anchored, so `-ignore /secrets` turned into a basename rule and dropped every "secrets" directory in the tree from the scan, not just the one at the root. --- cmd/portcullis-scan/ignore.go | 17 ++++++++++------- cmd/portcullis-scan/ignore_test.go | 20 ++++++++++++++------ cmd/portcullis-scan/main.go | 13 +++++++------ 3 files changed, 31 insertions(+), 19 deletions(-) diff --git a/cmd/portcullis-scan/ignore.go b/cmd/portcullis-scan/ignore.go index 34e18c6..898bf7b 100644 --- a/cmd/portcullis-scan/ignore.go +++ b/cmd/portcullis-scan/ignore.go @@ -15,11 +15,11 @@ import ( // - "**" matches any run of characters, including separators // - "[...]" character class (passed through to the regexp engine) // -// A pattern containing "/" is anchored to the scan root and matched -// against the slash-separated relative path; otherwise it matches -// the basename of every entry. A trailing "/" restricts the rule to -// directories. A "X/**" rule also prunes "X" itself so its subtree -// is not walked. +// A pattern with a "/" at its start or in its middle is anchored to +// the scan root and matched against the slash-separated relative +// path; otherwise it matches the basename of every entry. A trailing +// "/" restricts the rule to directories. A "X/**" rule also prunes +// "X" itself so its subtree is not walked. type ignoreMatcher struct { rules []ignoreRule } @@ -39,11 +39,14 @@ func newIgnoreMatcher(patterns []string) (*ignoreMatcher, error) { } dirOnly := strings.HasSuffix(p, "/") p = strings.TrimSuffix(p, "/") - p = strings.TrimPrefix(p, "/") + // A leading "/" anchors a pattern even when it has no other + // separator: "/build" is the root's build, not every build + // in the tree. + p, anchored := strings.CutPrefix(p, "/") if p == "" { continue } - pathMode := strings.Contains(p, "/") + pathMode := anchored || strings.Contains(p, "/") re, err := compileGlob(p) if err != nil { return nil, fmt.Errorf("invalid ignore pattern %q: %w", raw, err) diff --git a/cmd/portcullis-scan/ignore_test.go b/cmd/portcullis-scan/ignore_test.go index b26e869..9b74332 100644 --- a/cmd/portcullis-scan/ignore_test.go +++ b/cmd/portcullis-scan/ignore_test.go @@ -72,6 +72,10 @@ func TestRunIgnoreFlagIsRepeatable(t *testing.T) { assert.NotContains(t, out, "leaky_test.go") } +// TestRunIgnoresAnchoredPathPattern: as in gitignore, a "/" at the +// start or in the middle of a pattern anchors it to the scan root. +// Without the anchor "/secrets" would silently drop every "secrets" +// directory in the tree from the scan. func TestRunIgnoresAnchoredPathPattern(t *testing.T) { t.Parallel() @@ -83,13 +87,13 @@ func TestRunIgnoresAnchoredPathPattern(t *testing.T) { require.NoError(t, os.WriteFile(filepath.Join(root, "secrets", "g.txt"), []byte(githubPAT), 0o644)) - var stdout, stderr bytes.Buffer - code := run([]string{"-ignore", "secrets/**", root}, &stdout, &stderr) + for _, pattern := range []string{"secrets/**", "/secrets/**", "/secrets", "/secrets/"} { + var stdout, stderr bytes.Buffer + code := run([]string{"-ignore", pattern, root}, &stdout, &stderr) - assert.Equal(t, exitFound, code) - out := stdout.String() - assert.Contains(t, out, filepath.Join("a", "secrets", "f.txt")) - assert.NotContains(t, out, filepath.Join("secrets", "g.txt")) + assert.Equal(t, exitFound, code, pattern) + assert.Equal(t, filepath.Join("a", "secrets", "f.txt")+":1:1: "+githubPAT+"\n", stdout.String(), pattern) + } } func TestRunRejectsInvalidIgnorePattern(t *testing.T) { @@ -125,6 +129,10 @@ func TestIgnoreMatcher(t *testing.T) { {"double star prefix at root", []string{"**/foo.txt"}, "foo.txt", "foo.txt", false, true}, {"single star is single segment", []string{"a/*.go"}, "a/b/c.go", "c.go", false, false}, {"question mark", []string{"?.go"}, "a.go", "a.go", false, true}, + {"leading slash at root", []string{"/build"}, "build", "build", true, true}, + {"leading slash is anchored", []string{"/build"}, "src/build", "build", true, false}, + {"leading slash with star", []string{"/*.env"}, "sub/prod.env", "prod.env", false, false}, + {"leading slash dir-only on file", []string{"/build/"}, "build", "build", false, false}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { diff --git a/cmd/portcullis-scan/main.go b/cmd/portcullis-scan/main.go index 249a543..f02d57f 100644 --- a/cmd/portcullis-scan/main.go +++ b/cmd/portcullis-scan/main.go @@ -24,12 +24,13 @@ // -binary also scan binary files (default: skipped, like ripgrep). // A file is considered binary when its first 8 KiB // contains a NUL byte. -// -ignore glob pattern to skip; repeatable. Patterns containing -// '/' are anchored to the scan root, others match -// basenames. '**' matches any run of characters, -// including separators. Trailing '/' restricts the -// rule to directories. Examples: -ignore '*_test.go' -// -ignore 'vendor/**' -ignore 'node_modules/' +// -ignore glob pattern to skip; repeatable. Patterns with a +// '/' at the start or in the middle are anchored to +// the scan root, others match basenames. '**' matches +// any run of characters, including separators. +// Trailing '/' restricts the rule to directories. +// Examples: -ignore '*_test.go' -ignore 'vendor/**' +// -ignore 'node_modules/' -ignore '/build' // // Exit codes: // From b6debabd268a0d18b1243ef44ef55ea34cefd6a9 Mon Sep 17 00:00:00 2001 From: David Gageot Date: Fri, 20 Nov 2026 16:35:58 +0100 Subject: [PATCH 07/14] test: make the Bedrock known-token case exercise the Bedrock rule The fixture was `ABSK` plus filler, which the Bedrock validator rejects; the case only passed because `SK` + 32 hex characters trips the Twilio rule. Use a body that decodes to a Bedrock envelope. --- portcullis_test.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/portcullis_test.go b/portcullis_test.go index 62bfc43..1309b15 100644 --- a/portcullis_test.go +++ b/portcullis_test.go @@ -1,6 +1,7 @@ package portcullis_test import ( + "encoding/base64" "strings" "testing" "time" @@ -115,7 +116,9 @@ func TestContainsRecognisesKnownTokens(t *testing.T) { {"gitlab_runner_authentication_token", "glrt-" + strings.Repeat("a", 20)}, {"gitlab_scim_token", "glsoat-" + strings.Repeat("a", 20)}, {"gitlab_pat_routable", "glpat-" + strings.Repeat("a", 50) + ".aa1234567"}, - {"aws_bedrock_long_lived_key", "ABSK" + strings.Repeat("a", 200)}, + // The body has to decode to a Bedrock envelope: a run of + // filler after `ABSK` only passes by tripping the Twilio rule. + {"aws_bedrock_long_lived_key", "AB" + "SK" + base64.StdEncoding.EncodeToString([]byte("BedrockAPIKey-"+strings.Repeat("A", 82)))}, {"aws_bedrock_short_lived_key", "bedrock-api-key-" + "YmVkcm9jay5hbWF6b25hd3MuY29t" + strings.Repeat("a", 60)}, {"heroku_api_key_v2", "HRKU-AA" + strings.Repeat("a", 58)}, {"azure_ad_client_secret", "abc" + "1Q~" + strings.Repeat("a", 33)}, From 87ed3b9ed1ea73c0aab9fe28b871e98b43869bfb Mon Sep 17 00:00:00 2001 From: David Gageot Date: Fri, 20 Nov 2026 17:10:07 +0100 Subject: [PATCH 08/14] perf: gate the three PII rules with one pass over the input MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The card, SSN and IBAN prefilters each scanned the whole input, which on clean text cost five times the Aho-Corasick scan itself. They all start at a digit, so one pass can look for the three shapes at once. Redact and Contains on a 9 kB clean input: 17µs -> 6µs. --- portcullis.go | 6 ++- prefilter_test.go | 63 +++++++++++++++++++++---- rules.go | 32 ++++++------- validators.go | 114 +++++++++++++++++++++++++++++----------------- 4 files changed, 146 insertions(+), 69 deletions(-) diff --git a/portcullis.go b/portcullis.go index e7ad19d..1b77c40 100644 --- a/portcullis.go +++ b/portcullis.go @@ -61,10 +61,11 @@ func findMatches(text string) []Match { if !ok { return nil } + var seen lazyShapes var matches []Match for i := range rs.rules { r := &rs.rules[i] - if !r.passes(found, text) { + if !r.passes(found, &seen, text) { continue } compiled := r.compile() @@ -136,9 +137,10 @@ func Contains(text string) bool { // Redact left in its place, so that shortcut is reserved for // marker-free input. markers := strings.Count(text, Marker) + var seen lazyShapes for i := range rs.rules { r := &rs.rules[i] - if !r.passes(found, text) { + if !r.passes(found, &seen, text) { continue } compiled := r.compile() diff --git a/prefilter_test.go b/prefilter_test.go index bf015c2..43992d3 100644 --- a/prefilter_test.go +++ b/prefilter_test.go @@ -1,6 +1,7 @@ package portcullis import ( + "math/rand" "strings" "testing" @@ -30,26 +31,71 @@ func referenceSeparatedDigitRun(text string, minDigits int, ws bool) bool { return false } -func FuzzSeparatedDigitRun(f *testing.F) { +func referenceIBANAnchor(text string) bool { + for i := 0; i+3 < len(text); i++ { + if isASCIILetter(text[i]) && isASCIILetter(text[i+1]) && + isASCIIDigit(text[i+2]) && isASCIIDigit(text[i+3]) { + return true + } + } + return false +} + +// referenceShapes spells out each shape on its own, one pass per +// shape, as the definition [scanShapes] has to agree with. +func referenceShapes(text string) (found shapes) { + if referenceSeparatedDigitRun(text, 13, true) { + found |= shapeCard + } + if referenceSeparatedDigitRun(text, 9, false) { + found |= shapeSSN + } + if referenceIBANAnchor(text) { + found |= shapeIBAN + } + return found +} + +func FuzzScanShapes(f *testing.F) { for _, value := range []string{ "", "clean prose", "123456789", "4111 1111 1111 1111", "1\t2\n3\r4\v5\f6-7 8 9", "1 234567890123", "é123\xff456789", "12345678", "123456789012", "1234567890123", "1234- 567890123", "1234\t\n567890123", "12345678-", "123456789012\t", + "GB82", "B82", "GB8", "G8B2", "12GB82", "1 GB82", "1234\tGB82 WEST", + "123 45\t6789", "12345\t6789 12345", "DE89 3704 0044 0532 0130 00 123-45-6789", } { f.Add(value) } f.Fuzz(func(t *testing.T, value string) { - for _, n := range []int{-1, 0, 1, 9, 13, 19} { - for _, ws := range []bool{false, true} { - assert.Equal(t, referenceSeparatedDigitRun(value, n, ws), hasSeparatedDigitRun(value, n, ws), - "input %q, threshold %d, whitespace %t", value, n, ws) + assert.Equal(t, referenceShapes(value), scanShapes(value), "input %q", value) + }) +} + +// TestScanShapesMatchesReference feeds [scanShapes] digit-heavy noise, +// the inputs where its single pass has the most state to get wrong. +func TestScanShapesMatchesReference(t *testing.T) { + t.Parallel() + + const alphabet = "0123456789- \t\n\r\v\fabZ.\xc3" + rng := rand.New(rand.NewSource(1)) + for range 200_000 { + buf := make([]byte, rng.Intn(48)) + for i := range buf { + if rng.Intn(3) == 0 { + buf[i] = alphabet[rng.Intn(len(alphabet))] + } else { + buf[i] = byte('0' + rng.Intn(10)) } } - }) + value := string(buf) + if want, got := referenceShapes(value), scanShapes(value); want != got { + t.Fatalf("input %q: want %03b, got %03b", value, want, got) + } + } } -func BenchmarkDigitPrefilter(b *testing.B) { +func BenchmarkScanShapes(b *testing.B) { for _, tc := range []struct { name, input string }{ @@ -62,8 +108,7 @@ func BenchmarkDigitPrefilter(b *testing.B) { b.Run(tc.name, func(b *testing.B) { b.ReportAllocs() for b.Loop() { - maybeCreditCard(tc.input) - maybeSSN(tc.input) + scanShapes(tc.input) } }) } diff --git a/rules.go b/rules.go index 4087420..8ba2092 100644 --- a/rules.go +++ b/rules.go @@ -39,12 +39,12 @@ type rule struct { keywords []string caseSensitive bool validator func(string) bool - // prefilter is a cheap necessary-condition byte scan for - // keywordless rules, which the AC pre-filter can't gate. When set, - // the rule's regex only runs on inputs the prefilter accepts; it - // must never reject an input the regex would match. Ignored for - // rules that carry keywords. - prefilter func(string) bool + // shape is a cheap necessary condition for keywordless rules, + // which the AC pre-filter can't gate. When set, the rule's regex + // only runs on inputs where [scanShapes] finds that shape; it + // must never be missing from an input the regex would match. + // Ignored for rules that carry keywords. + shape shapes } // asSecretGroup wraps a `?P…` fragment in a plain group so @@ -1990,7 +1990,7 @@ func rules() []rule { // not get redacted as cards. expression: `(?P\b(?:4\d{3}|5[1-5]\d{2}|2[2-7]\d{2}|3[47]\d{2}|6(?:011|5\d{2})|3(?:0[0-5]|[68]\d)\d)(?:[-\s]?\d){9,15}\b)`, validator: validLuhn, - prefilter: maybeCreditCard, + shape: shapeCard, }, { // iban. Mirrors memclaw's IBAN detector: broad ISO 13616 @@ -2000,7 +2000,7 @@ func rules() []rule { // checksum to avoid flagging arbitrary account-like strings. expression: `(?i)(?P\b[A-Z]{2}\d{2}(?:[ ]?[A-Z0-9]{4}){2,7}(?:[ ]?[A-Z0-9]{1,3})?\b)`, validator: validIBAN, - prefilter: maybeIBAN, + shape: shapeIBAN, }, { // us-ssn. Mirrors memclaw's national-ID rule for US Social @@ -2008,7 +2008,7 @@ func rules() []rule { // area / group / serial ranges (000, 666, 9xx, 00, 0000). expression: `(?P\b\d{3}[- ]?\d{2}[- ]?\d{4}\b)`, validator: validUSSSN, - prefilter: maybeSSN, + shape: shapeSSN, }, { @@ -2053,22 +2053,22 @@ type compiledRule struct { kwBits kwMask caseSensitive bool csKW []string - prefilter func(string) bool + shape shapes compile func() compiledMatch // memoised } // passes returns true when the rule should run its regex against -// text: keywordless rules run whenever their cheap [rule.prefilter] -// byte scan accepts the input (a necessary condition for the regex to -// match, so clean prose skips the expensive expression); otherwise +// text: keywordless rules run whenever the input has their cheap +// [rule.shape] (a necessary condition for the regex to match, so +// clean prose skips the expensive expression); otherwise // the AC pre-filter must have matched, and — for case-sensitive // rules — at least one keyword must appear in the original input // without the case-fold. The post-filter cost is one // [strings.Contains] per CS keyword, two orders of magnitude // cheaper than running the rule's regex over the whole file. -func (r *compiledRule) passes(found kwMask, text string) bool { +func (r *compiledRule) passes(found kwMask, seen *lazyShapes, text string) bool { if r.kwBits.empty() { - return r.prefilter == nil || r.prefilter(text) + return r.shape == 0 || seen.has(text, r.shape) } if !found.overlaps(r.kwBits) { return false @@ -2132,7 +2132,7 @@ var compiledRuleSet = sync.OnceValue(func() *ruleSet { kwBits: bits, caseSensitive: r.caseSensitive, csKW: r.keywords, - prefilter: r.prefilter, + shape: r.shape, compile: sync.OnceValue(func() compiledMatch { re := regexp.MustCompile(expr) return compiledMatch{re: re, secretIdx: re.SubexpIndex("secret"), validator: validator} diff --git a/validators.go b/validators.go index a7b3407..ab738ae 100644 --- a/validators.go +++ b/validators.go @@ -235,62 +235,92 @@ func validAWSBedrockLongLivedKey(token string) bool { // The keywordless PII rules (credit card, IBAN, SSN) can't hide // behind the Aho–Corasick keyword pre-filter, so without a cheap gate // their expensive regexes would run over every clean input. Each -// prefilter below is a necessary condition for its rule's regex to -// match — it may pass inputs the regex ultimately rejects, but it -// never rejects an input the regex would accept — so clean prose -// (which has no long digit runs) skips the regex entirely. - -// hasSeparatedDigitRun reports whether text contains a run of at -// least minDigits decimal digits in which consecutive digits are -// split by at most one separator byte. Separators are '-' and ' '; -// the wider ASCII-whitespace set (tab / newline / …) is folded in -// when ws is true, to mirror the card rule's [-\s] separator class. -func hasSeparatedDigitRun(text string, minDigits int, ws bool) bool { +// shape below is a necessary condition for its rule's regex to +// match — it may show up in inputs the regex ultimately rejects, but +// it is never missing from an input the regex would accept — so clean +// prose (which has no long digit runs) skips the regex entirely. +type shapes uint8 + +const ( + // shapeCard gates the card rule: its regex consumes at least 13 + // digits (a 4-digit issuer group plus 9+ more) in a run where + // consecutive digits are split by at most one byte of its [-\s] + // separator class. + shapeCard shapes = 1 << iota + // shapeSSN gates the SSN rule: its regex is 9 digits split by at + // most single '-'/' ' separators. + shapeSSN + // shapeIBAN gates the IBAN rule on its mandatory country-code + + // check-digit anchor: two ASCII letters immediately followed by + // two ASCII digits. + shapeIBAN + + allShapes = shapeCard | shapeSSN | shapeIBAN +) + +// scanShapes reports every shape text contains, in a single pass. +// They all need a digit, so prose is skipped byte by byte without +// keeping any state and only the digit runs get a closer look. +func scanShapes(text string) (found shapes) { for i := 0; i < len(text); { - // Skip prose without maintaining digit-run state for every byte. - for i < len(text) && (text[i] < '0' || text[i] > '9') { + for i < len(text) && !isASCIIDigit(text[i]) { i++ } - run := 0 - for i < len(text) && text[i] >= '0' && text[i] <= '9' { - run++ - if run >= minDigits { - return true + // text[i], if any, opens a digit run: the one place where + // letters can sit right before digits. + if i >= 2 && i+1 < len(text) && isASCIIDigit(text[i+1]) && + isASCIILetter(text[i-1]) && isASCIILetter(text[i-2]) { + found |= shapeIBAN + } + // Count the digits that at most one separator byte splits + // from the previous one. '-' and ' ' separate for both rules; + // the rest of ASCII whitespace only does for cards, so it + // starts the SSN count over. + card, ssn := 0, 0 + for i < len(text) && isASCIIDigit(text[i]) { + card++ + ssn++ + if card >= 13 { + found |= shapeCard + } + if ssn >= 9 { + found |= shapeSSN } i++ if i < len(text) { - c := text[i] - if c == '-' || c == ' ' || - (ws && (c == '\t' || c == '\n' || c == '\r' || c == '\v' || c == '\f')) { + switch text[i] { + case '-', ' ': + i++ + case '\t', '\n', '\r', '\v', '\f': + ssn = 0 i++ } } } + if found == allShapes { + break + } } - return false + return found } -// maybeCreditCard gates the card rule: its regex consumes at least 13 -// digits (a 4-digit issuer group plus 9+ more) in a single [-\s]- -// separated run. -func maybeCreditCard(text string) bool { return hasSeparatedDigitRun(text, 13, true) } - -// maybeSSN gates the SSN rule: its regex is 9 digits split by at most -// single '-'/' ' separators. -func maybeSSN(text string) bool { return hasSeparatedDigitRun(text, 9, false) } - -// maybeIBAN gates the IBAN rule on its mandatory country-code + -// check-digit anchor: two ASCII letters immediately followed by two -// ASCII digits. -func maybeIBAN(text string) bool { - for i := 0; i+3 < len(text); i++ { - if isASCIILetter(text[i]) && isASCIILetter(text[i+1]) && - text[i+2] >= '0' && text[i+2] <= '9' && - text[i+3] >= '0' && text[i+3] <= '9' { - return true - } +// lazyShapes runs [scanShapes] over an input at most once, and only +// if a keywordless rule is reached at all: [Contains] often returns +// on a keyworded rule before that. +type lazyShapes struct { + scanned bool + found shapes +} + +func (l *lazyShapes) has(text string, shape shapes) bool { + if !l.scanned { + l.found, l.scanned = scanShapes(text), true } - return false + return l.found&shape != 0 +} + +func isASCIIDigit(c byte) bool { + return c >= '0' && c <= '9' } func isASCIILetter(c byte) bool { From 6a0022a3d29f2ba9b3eea883014062e4abcc38aa Mon Sep 17 00:00:00 2001 From: David Gageot Date: Fri, 20 Nov 2026 17:11:10 +0100 Subject: [PATCH 09/14] fix: match IBAN letters as ASCII only, like the rule's gate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `(?i)[A-Z]` also matches `ſ` and the Kelvin sign, which the gate does not take for letters. An IBAN spelled with `ſ` was therefore redacted or not depending on whether the input held another letter-letter- digit-digit run somewhere else. Spell out both cases instead, and check that every shape is implied by its rule's regex. --- prefilter_test.go | 34 ++++++++++++++++++++++++++++++++++ rules.go | 5 ++++- 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/prefilter_test.go b/prefilter_test.go index 43992d3..a4a7f5a 100644 --- a/prefilter_test.go +++ b/prefilter_test.go @@ -95,6 +95,40 @@ func TestScanShapesMatchesReference(t *testing.T) { } } +// TestShapesAreNecessaryForTheirRules checks the claim every shape +// rests on: an input its rule's regex matches always has the shape. +// A gate stricter than the regex would drop matches, and do so +// depending on what else the input happens to contain. +func TestShapesAreNecessaryForTheirRules(t *testing.T) { + t.Parallel() + + rs := compiledRuleSet() + // Some letters and separators, `ſ`, which case-folds to an ASCII + // letter without being one, and mostly digits. + alphabet := []string{"A", "b", "Z", "ſ", "-", ".", " ", "\t", "\n"} + for d := '0'; d <= '9'; d++ { + alphabet = append(alphabet, string(d), string(d)) + } + rng := rand.New(rand.NewSource(1)) + for range 20_000 { + var b strings.Builder + for range rng.Intn(40) { + b.WriteString(alphabet[rng.Intn(len(alphabet))]) + } + text := b.String() + found := scanShapes(text) + for i := range rs.rules { + r := &rs.rules[i] + if r.shape == 0 || found&r.shape != 0 { + continue + } + if loc := r.compile().re.FindStringIndex(text); loc != nil { + t.Fatalf("rule %d matches %q in %q, which lacks the rule's shape", i, text[loc[0]:loc[1]], text) + } + } + } +} + func BenchmarkScanShapes(b *testing.B) { for _, tc := range []struct { name, input string diff --git a/rules.go b/rules.go index 8ba2092..a77de06 100644 --- a/rules.go +++ b/rules.go @@ -1998,7 +1998,10 @@ func rules() []rule { // would let a match run into the next line), gated by the // registered country code and length and the official mod-97 // checksum to avoid flagging arbitrary account-like strings. - expression: `(?i)(?P\b[A-Z]{2}\d{2}(?:[ ]?[A-Z0-9]{4}){2,7}(?:[ ]?[A-Z0-9]{1,3})?\b)`, + // Both cases are spelled out rather than folded with `(?i)`, + // which would also let `ſ` and the Kelvin sign in as letters, + // past the ASCII ones [shapeIBAN] looks for. + expression: `(?P\b[A-Za-z]{2}\d{2}(?:[ ]?[A-Za-z0-9]{4}){2,7}(?:[ ]?[A-Za-z0-9]{1,3})?\b)`, validator: validIBAN, shape: shapeIBAN, }, From 8e4b1b28bf5b6b44451aac38fba79eaf22ae2221 Mon Sep 17 00:00:00 2001 From: David Gageot Date: Fri, 20 Nov 2026 17:12:21 +0100 Subject: [PATCH 10/14] docs: document Find, the []byte variants and portcullis-scan The README and package doc still described the API as Contains and Redact only, and never mentioned the CLI. The performance section predated the checksum validators and PII rules: refresh its numbers and drop the claims that no longer hold. --- README.md | 62 +++++++++++++++++++++++++++++++++++++++++-------------- doc.go | 15 +++++++------- 2 files changed, 54 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index 41542f6..1260220 100644 --- a/README.md +++ b/README.md @@ -47,12 +47,40 @@ const Marker = "[REDACTED]" func Contains(text string) bool // detect func Redact(text string) string // scrub +func Find(text string) []Match // locate + +// Same as above for callers holding a []byte, without copying it. +// Each Match.Value then aliases b: leave b alone while they are in use. +func ContainsBytes(b []byte) bool +func FindBytes(b []byte) []Match + +type Match struct { + Start, End int // byte offsets into the input + Value string // the input between them +} ``` -Both functions are safe for concurrent use. The compiled rule set and +All of them are safe for concurrent use. The compiled rule set and its Aho–Corasick automaton are built once on first call and shared across goroutines. +## Command line + +`portcullis-scan` runs the same detection over a file or a directory +tree and prints one `path:line:col: value` line per secret: + +```sh +go install github.com/docker/portcullis/cmd/portcullis-scan@latest + +portcullis-scan -ignore 'vendor/**' -ignore '*_test.go' . +``` + +It exits with 0 when nothing was found, 1 when a secret was found and +2 on a usage error or an unreadable file, so it can gate a CI job. +Its output holds the secrets themselves: where logs are kept, discard +it and go by the exit status. Binary files and files over 10 MiB are +skipped unless `-binary` and `-max-size` say otherwise. + ## What it detects The built-in catalogue covers ~240 patterns spanning: @@ -126,28 +154,30 @@ Go version specified in go.mod (Go 1.27.4) on every PR ## Performance -On a typical clean input the cost is dominated by a single linear scan -over the bytes; the regex engine is never invoked. With a secret, only -the rules whose keywords are present run. Both functions allocate -only when text actually changes. +An input with none of the rules' keywords and none of the shapes the +PII rules start from (nine or more digits in a row, single dashes or +spaces aside, or two letters right before two digits) never reaches a +regex and, once the rule set is built, allocates nothing. Its cost is +that of scanning the bytes for keywords (Aho–Corasick), then for +those shapes. Otherwise only the rules whose keyword or shape is +present run their regex. On an Apple M4 Max scrubbing a 9000-byte clean payload and a secret-bearing 1.5 KB payload: ``` -BenchmarkAhoScanCleanInput-16 257198 4823 ns/op 1865.95 MB/s 0 B/op 0 allocs/op -BenchmarkAhoScanWithKeyword-16 1251084 956.0 ns/op 1636.96 MB/s 0 B/op 0 allocs/op -BenchmarkRedactCleanInput-16 228036 4803 ns/op 0 B/op 0 allocs/op -BenchmarkContainsCleanInput-16 233336 4782 ns/op 0 B/op 0 allocs/op -BenchmarkRedactWithSecret-16 565590 2094 ns/op 1585 B/op 2 allocs/op -BenchmarkContainsWithSecret-16 912943 1312 ns/op 0 B/op 0 allocs/op +BenchmarkAhoScanCleanInput-16 427100 2832 ns/op 3177.96 MB/s 0 B/op 0 allocs/op +BenchmarkAhoScanWithKeyword-16 1760301 683.8 ns/op 2288.76 MB/s 0 B/op 0 allocs/op +BenchmarkRedactCleanInput-16 200227 6089 ns/op 0 B/op 0 allocs/op +BenchmarkContainsCleanInput-16 203072 6016 ns/op 0 B/op 0 allocs/op +BenchmarkRedactWithSecret-16 418003 2835 ns/op 1717 B/op 7 allocs/op +BenchmarkContainsWithSecret-16 745269 1609 ns/op 129 B/op 5 allocs/op ``` -The AC scan dominates the clean-input path — `Redact` and `Contains` -add no measurable overhead on top of it because the rule loop -short-circuits on an empty keyword mask. `Contains` allocates zero -bytes even on a secret-bearing input: it only needs the regex -`MatchString`, which uses pooled state machines internally. +`Contains` returns on the first rule that matches, so on a +secret-bearing input it usually skips the shape scan and the rest of +the catalogue. `Redact` has to visit every rule, and allocates for +the matches it finds and the redacted copy. ## Provenance diff --git a/doc.go b/doc.go index 1de0f02..fdc5ed9 100644 --- a/doc.go +++ b/doc.go @@ -1,18 +1,19 @@ // Package portcullis detects and redacts API tokens, cloud // credentials, and other secret material in arbitrary text. // -// [Contains] reports whether any rule matches the input; -// [Redact] replaces every detected secret span with [Marker] -// while preserving the surrounding text. Both are safe for -// concurrent use and idempotent. +// [Contains] reports whether any rule matches the input; [Find] +// returns the detected secret spans, the longest of those that +// overlap; [Redact] replaces every detected span with [Marker] +// while preserving the surrounding text. All three are safe for +// concurrent use, and [Redact] is idempotent. // // # Performance // // Detection runs in O(len(text)) per rule via Go's RE2-based // [regexp] engine, gated by an Aho–Corasick keyword pre-filter so -// clean inputs typically don't compile or run any regex. Memory -// allocations are zero on a clean input and small (a few hundred -// bytes) on a secret-bearing one. +// clean inputs typically don't compile or run any regex, and then +// allocate nothing. On a secret-bearing input [Redact] allocates +// for the matches it finds and the redacted copy. // // # Caller responsibilities // From 025cb21aadd66a3675a70809ed5c4940722b7d15 Mon Sep 17 00:00:00 2001 From: David Gageot Date: Fri, 20 Nov 2026 17:12:33 +0100 Subject: [PATCH 11/14] docs: fix the README example, which no longer redacted anything The snippet at the top of the README shows a GitHub token with an invalid checksum: since tokens are checksum-validated, Redact returns it untouched and Contains reports false. Show an AWS access key ID instead, and pin the snippet's inputs and results in a test so it can't go stale again. --- README.md | 8 ++++---- portcullis_test.go | 27 +++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 1260220..a7ffd17 100644 --- a/README.md +++ b/README.md @@ -8,11 +8,11 @@ credentials, and other secret material in arbitrary text. ```go import "github.com/docker/portcullis" -clean := portcullis.Redact("Run this with token=ghp_1234567890abcdef1234567890abcdef1234 please.") -// → "Run this with token=[REDACTED] please." +clean := portcullis.Redact("Run this with AWS_ACCESS_KEY_ID=AKIARZPUZDIKQEXAMPLE please.") +// → "Run this with AWS_ACCESS_KEY_ID=[REDACTED] please." -portcullis.Contains("not a secret") // false -portcullis.Contains("token=ghp_1234567890abcdef1234567890abcdef1234") // true +portcullis.Contains("not a secret") // false +portcullis.Contains("AWS_ACCESS_KEY_ID=AKIARZPUZDIKQEXAMPLE") // true ``` ## Why diff --git a/portcullis_test.go b/portcullis_test.go index 1309b15..b940143 100644 --- a/portcullis_test.go +++ b/portcullis_test.go @@ -2,6 +2,8 @@ package portcullis_test import ( "encoding/base64" + "os" + "strconv" "strings" "testing" "time" @@ -626,6 +628,31 @@ func TestRedactDoesNotSwallowAdjacentTextAfterPrefixedTokens(t *testing.T) { } } +// TestREADMEExample checks the inputs and results shown at the top of +// the README, the first thing a new user pastes. It once kept showing +// a GitHub token that the checksum validator had stopped accepting. +func TestREADMEExample(t *testing.T) { + t.Parallel() + + const ( + in = "Run this with AWS_ACCESS_KEY_ID=AKIARZPUZDIKQEXAMPLE please." + out = "Run this with AWS_ACCESS_KEY_ID=[REDACTED] please." + clean = "not a secret" + leaking = "AWS_ACCESS_KEY_ID=AKIARZPUZDIKQEXAMPLE" + ) + readme, err := os.ReadFile("README.md") + require.NoError(t, err) + for _, literal := range []string{in, out, clean, leaking} { + if quoted := strconv.Quote(literal); !strings.Contains(string(readme), quoted) { + t.Fatalf("README.md no longer shows %s: update this test along with it", quoted) + } + } + + assert.Equal(t, out, portcullis.Redact(in)) + assert.False(t, portcullis.Contains(clean)) + assert.True(t, portcullis.Contains(leaking)) +} + // TestContainsIgnoresHarmlessText: pure digit strings, plain English, // and the empty string must never trip detection. func TestContainsIgnoresHarmlessText(t *testing.T) { From 15d7b7993bd5df572f43eaf5190714099ab880bb Mon Sep 17 00:00:00 2001 From: David Gageot Date: Fri, 20 Nov 2026 17:12:45 +0100 Subject: [PATCH 12/14] docs: put the compiledRule comment on compiledRule It sat on compiledMatch, the type declared right above, and still said a clean input runs the PII expressions, which their shape gate now prevents. --- rules.go | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/rules.go b/rules.go index a77de06..7645899 100644 --- a/rules.go +++ b/rules.go @@ -2025,19 +2025,9 @@ func rules() []rule { } } -// compiledRule is the runtime form of a [rule]: its keywords are -// folded into a [kwMask] over the catalogue's shared keyword index, -// and its regex is compiled lazily on first match. A clean input — -// the overwhelmingly common case — only runs the small set of -// keywordless validator-gated PII expressions; all keyworded secret -// rules still stay behind the AC pre-filter. -// -// caseSensitive, when true, requires the post-filter step (see -// [compiledRule.passes]) to find at least one of [csKW] verbatim -// in the original-case input — not the case-folded form the AC -// already matched. This avoids running the regex on every file -// that happens to contain a folded form of a CS keyword (e.g. the -// English bigrams `mt`/`nd`/`od` for Discord, `sk` for Twilio). +// compiledMatch is what [compiledRule.compile] yields: the rule's +// regex, the index of its `secret` subgroup (-1 without one) and its +// optional validator. type compiledMatch struct { re *regexp.Regexp secretIdx int @@ -2052,6 +2042,20 @@ func (c *compiledMatch) accepts(value string) bool { return value != Marker && (c.validator == nil || c.validator(value)) } +// compiledRule is the runtime form of a [rule]: its keywords are +// folded into a [kwMask] over the catalogue's shared keyword index, +// and its regex is compiled lazily, the first time an input gets +// past the rule's gate. An input with no keyword and no PII shape — +// the overwhelmingly common case — runs no expression at all: the +// keyworded secret rules stay behind the AC pre-filter and the +// keywordless PII ones behind their [rule.shape]. +// +// caseSensitive, when true, requires the post-filter step (see +// [compiledRule.passes]) to find at least one of [csKW] verbatim +// in the original-case input — not the case-folded form the AC +// already matched. This avoids running the regex on every file +// that happens to contain a folded form of a CS keyword (e.g. the +// English bigrams `mt`/`nd`/`od` for Discord, `sk` for Twilio). type compiledRule struct { kwBits kwMask caseSensitive bool From 4e3732dafc91465a3095f4ffbbbffd3b8325e7b6 Mon Sep 17 00:00:00 2001 From: David Gageot Date: Fri, 20 Nov 2026 16:51:27 +0100 Subject: [PATCH 13/14] simplify: drop a Bedrock key length check that cannot fail A token of 132 characters always has a length that is a multiple of 4. --- validators.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validators.go b/validators.go index ab738ae..9493bf5 100644 --- a/validators.go +++ b/validators.go @@ -222,7 +222,7 @@ func validCloudflareAPIKey(token string) bool { } func validAWSBedrockLongLivedKey(token string) bool { - if len(token) != 132 || len(token)%4 != 0 { + if len(token) != 132 { return false } decoded, err := base64.StdEncoding.DecodeString(token[4:]) From f2890f692970572bbb2bb6410bdfb31ce9b1c469 Mon Sep 17 00:00:00 2001 From: David Gageot Date: Fri, 20 Nov 2026 17:37:25 +0100 Subject: [PATCH 14/14] fix(scan): keep a match on one line when its path has a line break Line breaks were collapsed in the matched value but not in the path, so a file name holding one split its match over two output lines, the second free to look like a match of its own. --- cmd/portcullis-scan/main.go | 10 ++++++---- cmd/portcullis-scan/main_test.go | 21 +++++++++++++++++++++ 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/cmd/portcullis-scan/main.go b/cmd/portcullis-scan/main.go index f02d57f..ab47fdd 100644 --- a/cmd/portcullis-scan/main.go +++ b/cmd/portcullis-scan/main.go @@ -7,8 +7,9 @@ // ::: // // Paths are relative to the scan root. Newlines and carriage returns -// inside a matched value are collapsed to spaces so each match -// remains on a single line; this only affects display, not detection. +// inside a path or a matched value are collapsed to spaces so each +// match remains on a single line; this only affects display, not +// detection. // Files are scanned in parallel; output order matches the walker's // lexical order, so two runs over the same tree produce identical // output regardless of worker count or scheduling. @@ -330,6 +331,7 @@ func scanFileBytes(path, root string, maxSize int64, scanBinary bool) ([]byte, e if rel, relErr := filepath.Rel(root, path); relErr == nil && rel != "." { display = rel } + display = sanitizeValue(display) var b strings.Builder // Matches arrive in ascending Start order, so line/column are // tracked with one cursor instead of rescanning data from the @@ -415,8 +417,8 @@ func readIfScannable(path string, maxSize int64, scanBinary bool) (data []byte, var valueSanitizer = strings.NewReplacer("\r\n", " ", "\n", " ", "\r", " ") -// sanitizeValue collapses CR / LF in v so a multi-line match (e.g. a -// PEM block) stays on a single output line. +// sanitizeValue collapses CR / LF in v, a matched value or a path, so +// a multi-line match (e.g. a PEM block) stays on a single output line. func sanitizeValue(v string) string { if !strings.ContainsAny(v, "\r\n") { return v diff --git a/cmd/portcullis-scan/main_test.go b/cmd/portcullis-scan/main_test.go index f4fa4ec..4b9a9f4 100644 --- a/cmd/portcullis-scan/main_test.go +++ b/cmd/portcullis-scan/main_test.go @@ -5,6 +5,7 @@ import ( "errors" "os" "path/filepath" + "runtime" "strings" "testing" @@ -216,6 +217,26 @@ func TestRunSanitisesMultilineSecrets(t *testing.T) { assert.NotContains(t, strings.TrimSuffix(out, "\n"), "\n") } +// TestRunKeepsPathOnOneLine: a file name can hold a line break too, +// and could otherwise pass the rest of its name off as an output line +// of its own. +func TestRunKeepsPathOnOneLine(t *testing.T) { + t.Parallel() + if runtime.GOOS == "windows" { + t.Skip("file names cannot contain line breaks") + } + + root := t.TempDir() + name := "a\nb.env:1:1: forged" + require.NoError(t, os.WriteFile(filepath.Join(root, name), []byte("TOKEN="+githubPAT+"\n"), 0o644)) + + var stdout, stderr bytes.Buffer + code := run([]string{root}, &stdout, &stderr) + + assert.Equal(t, exitFound, code) + assert.Equal(t, "a b.env:1:1: forged:1:7: "+githubPAT+"\n", stdout.String()) +} + func TestRunSkipsBinaryFiles(t *testing.T) { t.Parallel()