diff --git a/README.md b/README.md index 41542f6..a7ffd17 100644 --- a/README.md +++ b/README.md @@ -8,11 +8,11 @@ credentials, and other secret material in arbitrary text. ```go import "github.com/docker/portcullis" -clean := portcullis.Redact("Run this with token=ghp_1234567890abcdef1234567890abcdef1234 please.") -// → "Run this with token=[REDACTED] please." +clean := portcullis.Redact("Run this with AWS_ACCESS_KEY_ID=AKIARZPUZDIKQEXAMPLE please.") +// → "Run this with AWS_ACCESS_KEY_ID=[REDACTED] please." -portcullis.Contains("not a secret") // false -portcullis.Contains("token=ghp_1234567890abcdef1234567890abcdef1234") // true +portcullis.Contains("not a secret") // false +portcullis.Contains("AWS_ACCESS_KEY_ID=AKIARZPUZDIKQEXAMPLE") // true ``` ## Why @@ -47,12 +47,40 @@ const Marker = "[REDACTED]" func Contains(text string) bool // detect func Redact(text string) string // scrub +func Find(text string) []Match // locate + +// Same as above for callers holding a []byte, without copying it. +// Each Match.Value then aliases b: leave b alone while they are in use. +func ContainsBytes(b []byte) bool +func FindBytes(b []byte) []Match + +type Match struct { + Start, End int // byte offsets into the input + Value string // the input between them +} ``` -Both functions are safe for concurrent use. The compiled rule set and +All of them are safe for concurrent use. The compiled rule set and its Aho–Corasick automaton are built once on first call and shared across goroutines. +## Command line + +`portcullis-scan` runs the same detection over a file or a directory +tree and prints one `path:line:col: value` line per secret: + +```sh +go install github.com/docker/portcullis/cmd/portcullis-scan@latest + +portcullis-scan -ignore 'vendor/**' -ignore '*_test.go' . +``` + +It exits with 0 when nothing was found, 1 when a secret was found and +2 on a usage error or an unreadable file, so it can gate a CI job. +Its output holds the secrets themselves: where logs are kept, discard +it and go by the exit status. Binary files and files over 10 MiB are +skipped unless `-binary` and `-max-size` say otherwise. + ## What it detects The built-in catalogue covers ~240 patterns spanning: @@ -126,28 +154,30 @@ Go version specified in go.mod (Go 1.27.4) on every PR ## Performance -On a typical clean input the cost is dominated by a single linear scan -over the bytes; the regex engine is never invoked. With a secret, only -the rules whose keywords are present run. Both functions allocate -only when text actually changes. +An input with none of the rules' keywords and none of the shapes the +PII rules start from (nine or more digits in a row, single dashes or +spaces aside, or two letters right before two digits) never reaches a +regex and, once the rule set is built, allocates nothing. Its cost is +that of scanning the bytes for keywords (Aho–Corasick), then for +those shapes. Otherwise only the rules whose keyword or shape is +present run their regex. On an Apple M4 Max scrubbing a 9000-byte clean payload and a secret-bearing 1.5 KB payload: ``` -BenchmarkAhoScanCleanInput-16 257198 4823 ns/op 1865.95 MB/s 0 B/op 0 allocs/op -BenchmarkAhoScanWithKeyword-16 1251084 956.0 ns/op 1636.96 MB/s 0 B/op 0 allocs/op -BenchmarkRedactCleanInput-16 228036 4803 ns/op 0 B/op 0 allocs/op -BenchmarkContainsCleanInput-16 233336 4782 ns/op 0 B/op 0 allocs/op -BenchmarkRedactWithSecret-16 565590 2094 ns/op 1585 B/op 2 allocs/op -BenchmarkContainsWithSecret-16 912943 1312 ns/op 0 B/op 0 allocs/op +BenchmarkAhoScanCleanInput-16 427100 2832 ns/op 3177.96 MB/s 0 B/op 0 allocs/op +BenchmarkAhoScanWithKeyword-16 1760301 683.8 ns/op 2288.76 MB/s 0 B/op 0 allocs/op +BenchmarkRedactCleanInput-16 200227 6089 ns/op 0 B/op 0 allocs/op +BenchmarkContainsCleanInput-16 203072 6016 ns/op 0 B/op 0 allocs/op +BenchmarkRedactWithSecret-16 418003 2835 ns/op 1717 B/op 7 allocs/op +BenchmarkContainsWithSecret-16 745269 1609 ns/op 129 B/op 5 allocs/op ``` -The AC scan dominates the clean-input path — `Redact` and `Contains` -add no measurable overhead on top of it because the rule loop -short-circuits on an empty keyword mask. `Contains` allocates zero -bytes even on a secret-bearing input: it only needs the regex -`MatchString`, which uses pooled state machines internally. +`Contains` returns on the first rule that matches, so on a +secret-bearing input it usually skips the shape scan and the rest of +the catalogue. `Redact` has to visit every rule, and allocates for +the matches it finds and the redacted copy. ## Provenance diff --git a/aho.go b/aho.go index 3d271c2..6c4acd8 100644 --- a/aho.go +++ b/aho.go @@ -4,7 +4,7 @@ package portcullis // which patterns occurred in a scanned input and which keywords each // rule subscribes to. Storing it as a small array keeps every test // branch-free; the cap of len(kwMask)*64 indices (currently 320) -// accommodates today's catalogue of ~285 unique keywords with limited +// accommodates today's catalogue of ~295 unique keywords with limited // remaining headroom for future rules (an overflow trips a // deterministic panic in [buildAhoCorasick]). type kwMask [5]uint64 diff --git a/cmd/portcullis-scan/ignore.go b/cmd/portcullis-scan/ignore.go index 34e18c6..898bf7b 100644 --- a/cmd/portcullis-scan/ignore.go +++ b/cmd/portcullis-scan/ignore.go @@ -15,11 +15,11 @@ import ( // - "**" matches any run of characters, including separators // - "[...]" character class (passed through to the regexp engine) // -// A pattern containing "/" is anchored to the scan root and matched -// against the slash-separated relative path; otherwise it matches -// the basename of every entry. A trailing "/" restricts the rule to -// directories. A "X/**" rule also prunes "X" itself so its subtree -// is not walked. +// A pattern with a "/" at its start or in its middle is anchored to +// the scan root and matched against the slash-separated relative +// path; otherwise it matches the basename of every entry. A trailing +// "/" restricts the rule to directories. A "X/**" rule also prunes +// "X" itself so its subtree is not walked. type ignoreMatcher struct { rules []ignoreRule } @@ -39,11 +39,14 @@ func newIgnoreMatcher(patterns []string) (*ignoreMatcher, error) { } dirOnly := strings.HasSuffix(p, "/") p = strings.TrimSuffix(p, "/") - p = strings.TrimPrefix(p, "/") + // A leading "/" anchors a pattern even when it has no other + // separator: "/build" is the root's build, not every build + // in the tree. + p, anchored := strings.CutPrefix(p, "/") if p == "" { continue } - pathMode := strings.Contains(p, "/") + pathMode := anchored || strings.Contains(p, "/") re, err := compileGlob(p) if err != nil { return nil, fmt.Errorf("invalid ignore pattern %q: %w", raw, err) diff --git a/cmd/portcullis-scan/ignore_test.go b/cmd/portcullis-scan/ignore_test.go index b26e869..9b74332 100644 --- a/cmd/portcullis-scan/ignore_test.go +++ b/cmd/portcullis-scan/ignore_test.go @@ -72,6 +72,10 @@ func TestRunIgnoreFlagIsRepeatable(t *testing.T) { assert.NotContains(t, out, "leaky_test.go") } +// TestRunIgnoresAnchoredPathPattern: as in gitignore, a "/" at the +// start or in the middle of a pattern anchors it to the scan root. +// Without the anchor "/secrets" would silently drop every "secrets" +// directory in the tree from the scan. func TestRunIgnoresAnchoredPathPattern(t *testing.T) { t.Parallel() @@ -83,13 +87,13 @@ func TestRunIgnoresAnchoredPathPattern(t *testing.T) { require.NoError(t, os.WriteFile(filepath.Join(root, "secrets", "g.txt"), []byte(githubPAT), 0o644)) - var stdout, stderr bytes.Buffer - code := run([]string{"-ignore", "secrets/**", root}, &stdout, &stderr) + for _, pattern := range []string{"secrets/**", "/secrets/**", "/secrets", "/secrets/"} { + var stdout, stderr bytes.Buffer + code := run([]string{"-ignore", pattern, root}, &stdout, &stderr) - assert.Equal(t, exitFound, code) - out := stdout.String() - assert.Contains(t, out, filepath.Join("a", "secrets", "f.txt")) - assert.NotContains(t, out, filepath.Join("secrets", "g.txt")) + assert.Equal(t, exitFound, code, pattern) + assert.Equal(t, filepath.Join("a", "secrets", "f.txt")+":1:1: "+githubPAT+"\n", stdout.String(), pattern) + } } func TestRunRejectsInvalidIgnorePattern(t *testing.T) { @@ -125,6 +129,10 @@ func TestIgnoreMatcher(t *testing.T) { {"double star prefix at root", []string{"**/foo.txt"}, "foo.txt", "foo.txt", false, true}, {"single star is single segment", []string{"a/*.go"}, "a/b/c.go", "c.go", false, false}, {"question mark", []string{"?.go"}, "a.go", "a.go", false, true}, + {"leading slash at root", []string{"/build"}, "build", "build", true, true}, + {"leading slash is anchored", []string{"/build"}, "src/build", "build", true, false}, + {"leading slash with star", []string{"/*.env"}, "sub/prod.env", "prod.env", false, false}, + {"leading slash dir-only on file", []string{"/build/"}, "build", "build", false, false}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { diff --git a/cmd/portcullis-scan/main.go b/cmd/portcullis-scan/main.go index e774fe1..ab47fdd 100644 --- a/cmd/portcullis-scan/main.go +++ b/cmd/portcullis-scan/main.go @@ -7,8 +7,9 @@ // ::: // // Paths are relative to the scan root. Newlines and carriage returns -// inside a matched value are collapsed to spaces so each match -// remains on a single line; this only affects display, not detection. +// inside a path or a matched value are collapsed to spaces so each +// match remains on a single line; this only affects display, not +// detection. // Files are scanned in parallel; output order matches the walker's // lexical order, so two runs over the same tree produce identical // output regardless of worker count or scheduling. @@ -24,12 +25,13 @@ // -binary also scan binary files (default: skipped, like ripgrep). // A file is considered binary when its first 8 KiB // contains a NUL byte. -// -ignore glob pattern to skip; repeatable. Patterns containing -// '/' are anchored to the scan root, others match -// basenames. '**' matches any run of characters, -// including separators. Trailing '/' restricts the -// rule to directories. Examples: -ignore '*_test.go' -// -ignore 'vendor/**' -ignore 'node_modules/' +// -ignore glob pattern to skip; repeatable. Patterns with a +// '/' at the start or in the middle are anchored to +// the scan root, others match basenames. '**' matches +// any run of characters, including separators. +// Trailing '/' restricts the rule to directories. +// Examples: -ignore '*_test.go' -ignore 'vendor/**' +// -ignore 'node_modules/' -ignore '/build' // // Exit codes: // @@ -41,10 +43,12 @@ package main import ( "bytes" + "errors" "flag" "fmt" "io" "io/fs" + "math" "os" "path/filepath" "runtime" @@ -327,6 +331,7 @@ func scanFileBytes(path, root string, maxSize int64, scanBinary bool) ([]byte, e if rel, relErr := filepath.Rel(root, path); relErr == nil && rel != "." { display = rel } + display = sanitizeValue(display) var b strings.Builder // Matches arrive in ascending Start order, so line/column are // tracked with one cursor instead of rescanning data from the @@ -361,44 +366,59 @@ func readIfScannable(path string, maxSize int64, scanBinary bool) (data []byte, if err != nil { return nil, false, err } - size := st.Size() - if maxSize > 0 && size > maxSize { + if maxSize > 0 && st.Size() > maxSize { return nil, false, nil } - if scanBinary { - data, err = io.ReadAll(f) - if err != nil { - return nil, false, err - } - return data, true, nil + // That size is only a hint: a pipe (`<(git diff)`, a FIFO) reports + // 0 and a file being written to can outgrow it. So read to EOF + // rather than to that size, stopping one byte past maxSize so that + // content over the limit is still skipped. + var r io.Reader = f + if maxSize > 0 && maxSize < math.MaxInt64 { + r = io.LimitReader(f, maxSize+1) } // Sniff first 8 KiB for a NUL byte before committing to read // the rest. On large binaries (git packs, archives, prebuilt // executables) this avoids both the IO and the regex pass. - sniffN := min(int64(binarySniffSize), size) - sniff := make([]byte, sniffN) - if _, err = io.ReadFull(f, sniff); err != nil { + // + // The buffer starts one byte past the stat size, enough to see a + // small file end where it should, and only widens to the full + // window if there turns out to be more. + data = make([]byte, min(st.Size(), binarySniffSize-1)+1) + n, err := io.ReadFull(r, data) + if err == nil && n < binarySniffSize { + data = append(data, make([]byte, binarySniffSize-n)...) + var more int + more, err = io.ReadFull(r, data[n:]) + n += more + } + atEOF := errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) + if err != nil && !atEOF { return nil, false, err } - if bytes.IndexByte(sniff, 0) >= 0 { + data = data[:n] + if !scanBinary && bytes.IndexByte(data, 0) >= 0 { return nil, false, nil } - if size == sniffN { - return sniff, true, nil + if !atEOF { + rest, err := io.ReadAll(r) + if err != nil { + return nil, false, err + } + data = append(data, rest...) } - rest, err := io.ReadAll(f) - if err != nil { - return nil, false, err + if maxSize > 0 && int64(len(data)) > maxSize { + return nil, false, nil } - return append(sniff, rest...), true, nil + return data, true, nil } var valueSanitizer = strings.NewReplacer("\r\n", " ", "\n", " ", "\r", " ") -// sanitizeValue collapses CR / LF in v so a multi-line match (e.g. a -// PEM block) stays on a single output line. +// sanitizeValue collapses CR / LF in v, a matched value or a path, so +// a multi-line match (e.g. a PEM block) stays on a single output line. func sanitizeValue(v string) string { if !strings.ContainsAny(v, "\r\n") { return v diff --git a/cmd/portcullis-scan/main_test.go b/cmd/portcullis-scan/main_test.go index f4fa4ec..4b9a9f4 100644 --- a/cmd/portcullis-scan/main_test.go +++ b/cmd/portcullis-scan/main_test.go @@ -5,6 +5,7 @@ import ( "errors" "os" "path/filepath" + "runtime" "strings" "testing" @@ -216,6 +217,26 @@ func TestRunSanitisesMultilineSecrets(t *testing.T) { assert.NotContains(t, strings.TrimSuffix(out, "\n"), "\n") } +// TestRunKeepsPathOnOneLine: a file name can hold a line break too, +// and could otherwise pass the rest of its name off as an output line +// of its own. +func TestRunKeepsPathOnOneLine(t *testing.T) { + t.Parallel() + if runtime.GOOS == "windows" { + t.Skip("file names cannot contain line breaks") + } + + root := t.TempDir() + name := "a\nb.env:1:1: forged" + require.NoError(t, os.WriteFile(filepath.Join(root, name), []byte("TOKEN="+githubPAT+"\n"), 0o644)) + + var stdout, stderr bytes.Buffer + code := run([]string{root}, &stdout, &stderr) + + assert.Equal(t, exitFound, code) + assert.Equal(t, "a b.env:1:1: forged:1:7: "+githubPAT+"\n", stdout.String()) +} + func TestRunSkipsBinaryFiles(t *testing.T) { t.Parallel() diff --git a/cmd/portcullis-scan/stream_unix_test.go b/cmd/portcullis-scan/stream_unix_test.go new file mode 100644 index 0000000..d7645c2 --- /dev/null +++ b/cmd/portcullis-scan/stream_unix_test.go @@ -0,0 +1,82 @@ +//go:build unix && !aix && !solaris + +package main + +import ( + "bytes" + "os" + "path/filepath" + "strconv" + "syscall" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// newFIFO returns the path of a FIFO that yields content to its first +// reader. A FIFO stands in for any target whose size stat can't know +// up front, such as the pipe behind `portcullis-scan <(git diff)`. +func newFIFO(t *testing.T, content string) string { + t.Helper() + + fifo := filepath.Join(t.TempDir(), "stream") + require.NoError(t, syscall.Mkfifo(fifo, 0o600)) + written := make(chan error, 1) + go func() { + // Blocks until the scanner opens the FIFO. + written <- os.WriteFile(fifo, []byte(content), 0o600) + }() + t.Cleanup(func() { + select { + case err := <-written: + assert.NoError(t, err) + case <-time.After(10 * time.Second): + t.Error("the scanner never opened the FIFO") + } + }) + return fifo +} + +// TestRunScansStreamTarget: stat reports a pipe as 0 bytes long, which +// must not be taken to mean there is nothing to scan. -max-size applies +// to what the pipe actually delivers. +func TestRunScansStreamTarget(t *testing.T) { + t.Parallel() + + const content = "TOKEN=" + githubPAT + "\n" + atLimit := "-max-size=" + strconv.Itoa(len(content)) + + for _, tc := range []struct { + name string + flags []string + content string + found bool + }{ + {"default", nil, content, true}, + {"binary", []string{"-binary"}, content, true}, + {"at max size", []string{atLimit}, content, true}, + {"at max size, binary", []string{atLimit, "-binary"}, content, true}, + {"above max size", []string{atLimit}, content + "\n", false}, + {"above max size, binary", []string{atLimit, "-binary"}, content + "\n", false}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + fifo := newFIFO(t, tc.content) + + var stdout, stderr bytes.Buffer + code := run(append(tc.flags, fifo), &stdout, &stderr) + + assert.Empty(t, stderr.String()) + if tc.found { + assert.Equal(t, exitFound, code) + assert.Equal(t, fifo+":1:7: "+githubPAT+"\n", stdout.String()) + } else { + assert.Equal(t, exitClean, code) + assert.Empty(t, stdout.String()) + } + }) + } +} diff --git a/doc.go b/doc.go index 1de0f02..fdc5ed9 100644 --- a/doc.go +++ b/doc.go @@ -1,18 +1,19 @@ // Package portcullis detects and redacts API tokens, cloud // credentials, and other secret material in arbitrary text. // -// [Contains] reports whether any rule matches the input; -// [Redact] replaces every detected secret span with [Marker] -// while preserving the surrounding text. Both are safe for -// concurrent use and idempotent. +// [Contains] reports whether any rule matches the input; [Find] +// returns the detected secret spans, the longest of those that +// overlap; [Redact] replaces every detected span with [Marker] +// while preserving the surrounding text. All three are safe for +// concurrent use, and [Redact] is idempotent. // // # Performance // // Detection runs in O(len(text)) per rule via Go's RE2-based // [regexp] engine, gated by an Aho–Corasick keyword pre-filter so -// clean inputs typically don't compile or run any regex. Memory -// allocations are zero on a clean input and small (a few hundred -// bytes) on a secret-bearing one. +// clean inputs typically don't compile or run any regex, and then +// allocate nothing. On a secret-bearing input [Redact] allocates +// for the matches it finds and the redacted copy. // // # Caller responsibilities // diff --git a/edge_cases_test.go b/edge_cases_test.go index b1fcdab..cf5943f 100644 --- a/edge_cases_test.go +++ b/edge_cases_test.go @@ -65,12 +65,14 @@ func TestNoisyKeywordsFalsePositives(t *testing.T) { } } -// TestDiscordTokenPrefixes verifies all documented Discord bot token prefixes. +// TestDiscordTokenPrefixes verifies every prefix a Discord bot token +// can start with. The first segment is the base64 of the bot's numeric +// ID, so the leading decimal digit (1-9) fixes the first two +// characters. func TestDiscordTokenPrefixes(t *testing.T) { t.Parallel() - // Test all documented prefixes - prefixes := []string{"MT", "Mz", "ND", "NT", "Nz", "OD"} + prefixes := []string{"MT", "Mj", "Mz", "ND", "NT", "Nj", "Nz", "OD", "OT"} for _, prefix := range prefixes { t.Run("prefix_"+prefix, func(t *testing.T) { @@ -82,6 +84,18 @@ func TestDiscordTokenPrefixes(t *testing.T) { } } +// TestDiscordTokenNeedsItsPrefix: a prefix elsewhere in the input gets +// the rule past the keyword filter, but must not make a token out of +// a dotted identifier that doesn't start with one itself. +func TestDiscordTokenNeedsItsPrefix(t *testing.T) { + t.Parallel() + + in := "NOTE M" + strings.Repeat("a", 23) + "." + strings.Repeat("B", 6) + "." + strings.Repeat("C", 30) + + assert.False(t, portcullis.Contains(in)) + assert.Equal(t, in, portcullis.Redact(in)) +} + // TestConnectionStringContextPreservation verifies that MongoDB, Postgres, // and Azure Storage connection strings only redact the password/key portion. func TestConnectionStringContextPreservation(t *testing.T) { diff --git a/portcullis.go b/portcullis.go index 0e0761b..1b77c40 100644 --- a/portcullis.go +++ b/portcullis.go @@ -9,7 +9,9 @@ import ( // Marker replaces every detected secret span. Chosen so it doesn't // match any rule's keyword pre-filter — see TestMarkerIsNotASecret -// for the safety property that makes [Redact] idempotent. +// for the safety property that makes [Redact] idempotent. A marker +// sitting where a rule expects a secret (the password of an +// already-redacted connection string, say) is never reported as one. const Marker = "[REDACTED]" // Match describes a single secret span detected in an input. Start @@ -59,10 +61,11 @@ func findMatches(text string) []Match { if !ok { return nil } + var seen lazyShapes var matches []Match for i := range rs.rules { r := &rs.rules[i] - if !r.passes(found, text) { + if !r.passes(found, &seen, text) { continue } compiled := r.compile() @@ -72,7 +75,7 @@ func findMatches(text string) []Match { continue } value := text[s:e] - if compiled.validator != nil && !compiled.validator(value) { + if !compiled.accepts(value) { continue } matches = append(matches, Match{Start: s, End: e, Value: value}) @@ -130,21 +133,33 @@ func Contains(text string) bool { if !ok { return false } + // MatchString can't tell a secret from the marker a previous + // Redact left in its place, so that shortcut is reserved for + // marker-free input. + markers := strings.Count(text, Marker) + var seen lazyShapes for i := range rs.rules { r := &rs.rules[i] - if !r.passes(found, text) { + if !r.passes(found, &seen, text) { continue } compiled := r.compile() - if compiled.validator == nil { + if compiled.validator == nil && markers == 0 { if compiled.re.MatchString(text) { return true } continue } - for _, m := range compiled.re.FindAllStringSubmatchIndex(text, -1) { + // A validator can reject any number of matches. Without one + // only markers are rejected, one match per marker at most, so + // the match after those settles it. + limit := -1 + if compiled.validator == nil { + limit = markers + 1 + } + for _, m := range compiled.re.FindAllStringSubmatchIndex(text, limit) { s, e := redactSpan(m, compiled.secretIdx) - if compiled.validator(text[s:e]) { + if compiled.accepts(text[s:e]) { return true } } diff --git a/portcullis_test.go b/portcullis_test.go index 5561a82..b940143 100644 --- a/portcullis_test.go +++ b/portcullis_test.go @@ -1,6 +1,9 @@ package portcullis_test import ( + "encoding/base64" + "os" + "strconv" "strings" "testing" "time" @@ -115,7 +118,9 @@ func TestContainsRecognisesKnownTokens(t *testing.T) { {"gitlab_runner_authentication_token", "glrt-" + strings.Repeat("a", 20)}, {"gitlab_scim_token", "glsoat-" + strings.Repeat("a", 20)}, {"gitlab_pat_routable", "glpat-" + strings.Repeat("a", 50) + ".aa1234567"}, - {"aws_bedrock_long_lived_key", "ABSK" + strings.Repeat("a", 200)}, + // The body has to decode to a Bedrock envelope: a run of + // filler after `ABSK` only passes by tripping the Twilio rule. + {"aws_bedrock_long_lived_key", "AB" + "SK" + base64.StdEncoding.EncodeToString([]byte("BedrockAPIKey-"+strings.Repeat("A", 82)))}, {"aws_bedrock_short_lived_key", "bedrock-api-key-" + "YmVkcm9jay5hbWF6b25hd3MuY29t" + strings.Repeat("a", 60)}, {"heroku_api_key_v2", "HRKU-AA" + strings.Repeat("a", 58)}, {"azure_ad_client_secret", "abc" + "1Q~" + strings.Repeat("a", 33)}, @@ -277,6 +282,8 @@ func TestContainsRecognisesKnownTokens(t *testing.T) { assert.NotContainsf(t, out, tc.text, "raw secret must be gone after Redact: %q", out) assert.Containsf(t, out, portcullis.Marker, "redaction marker must appear in %q", out) + assert.Falsef(t, portcullis.Contains(out), + "redacted output must no longer trip Contains: %q", out) }) } } @@ -438,6 +445,133 @@ func TestRedactDoesNotSwallowAdjacentTextAfterSlackRotatingToken(t *testing.T) { "adjacent hostname must NOT be swallowed by the rotating-token regex: %q", out) } +// TestRedactCoversWholeSlackAccessToken pins the span of the +// slack-access-token rule. Slack tokens are dash-separated and the +// secret is the last segment (`xoxb---`), so +// a match that stops at the first dash leaves the secret in the +// output. +func TestRedactCoversWholeSlackAccessToken(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + token string + }{ + {"bot", "xoxb-" + strings.Repeat("1", 11) + "-" + strings.Repeat("2", 13) + "-" + strings.Repeat("a", 24)}, + {"user", "xoxp-" + strings.Repeat("1", 11) + "-" + strings.Repeat("2", 13) + "-" + strings.Repeat("3", 13) + "-" + strings.Repeat("f", 32)}, + {"session", "xoxs-" + strings.Repeat("1", 11) + "-" + strings.Repeat("2", 11) + "-" + strings.Repeat("3", 11) + "-" + strings.Repeat("f", 64)}, + {"single_segment", "xoxa-" + strings.Repeat("a", 24)}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + in := "SLACK_TOKEN=" + tc.token + " and the rest of the line" + + assert.Equal(t, "SLACK_TOKEN="+portcullis.Marker+" and the rest of the line", portcullis.Redact(in)) + matches := portcullis.Find(in) + require.Len(t, matches, 1) + assert.Equal(t, tc.token, matches[0].Value) + }) + } + + // The first segment still has to look like an ID, so the + // placeholder found in every Slack tutorial keeps passing through. + assert.False(t, portcullis.Contains("xoxb-your-token-here")) +} + +// TestSlackAccessTokenStopsAtItsLastSegment: the match must not run +// on into whatever is hyphenated onto a token, least of all into the +// prefix of a second token, which would then go undetected. +func TestSlackAccessTokenStopsAtItsLastSegment(t *testing.T) { + t.Parallel() + + user := "xoxp-" + strings.Repeat("1", 11) + "-" + strings.Repeat("2", 13) + "-" + strings.Repeat("3", 13) + "-" + strings.Repeat("f", 32) + bot := "xoxb-" + strings.Repeat("1", 11) + "-" + strings.Repeat("2", 13) + "-" + strings.Repeat("a", 24) + legacy := "xoxa-" + strings.Repeat("a", 24) + + assert.Equal(t, portcullis.Marker+"-retry-count", portcullis.Redact(user+"-retry-count")) + assert.Equal(t, portcullis.Marker+"-retry-count", portcullis.Redact(bot+"-retry-count")) + + for _, second := range []string{user, bot, legacy} { + matches := portcullis.Find(user + "-" + second) + require.Len(t, matches, 2) + assert.Equal(t, user, matches[0].Value) + assert.Equal(t, second, matches[1].Value) + } +} + +// TestRedactCoversGluedSlackAccessTokens: with no separator at all, +// the last segment of a token takes the `xoxb` of the next one. What +// is left of that one must not stay behind in clear. +func TestRedactCoversGluedSlackAccessTokens(t *testing.T) { + t.Parallel() + + user := "xoxp-" + strings.Repeat("1", 11) + "-" + strings.Repeat("2", 13) + "-" + strings.Repeat("3", 13) + "-" + strings.Repeat("f", 32) + bot := "xoxb-" + strings.Repeat("1", 11) + "-" + strings.Repeat("2", 13) + "-" + strings.Repeat("a", 24) + legacy := "xoxa-" + strings.Repeat("b", 24) + + for _, glued := range []string{bot + legacy, legacy + bot, user + bot + legacy, legacy + legacy} { + assert.Equal(t, "token="+portcullis.Marker+" end", portcullis.Redact("token="+glued+" end"), glued) + matches := portcullis.Find(glued) + require.Len(t, matches, 1, glued) + assert.Equal(t, glued, matches[0].Value) + } +} + +// TestRedactCoversWholeStripeKey pins the body length of the Stripe +// secret and restricted key rules. Current keys (`sk_live_51…`) carry +// a 99-character body, so a lower cap leaves the tail of the key in +// the output. +func TestRedactCoversWholeStripeKey(t *testing.T) { + t.Parallel() + + for _, prefix := range []string{"sk_live_", "sk_test_", "sk_prod_", "rk_live_", "rk_test_", "rk_prod_"} { + t.Run(prefix, func(t *testing.T) { + t.Parallel() + + key := prefix + "51" + strings.Repeat("aB3", 32) + "x" + in := "STRIPE_KEY=" + key + " and the rest of the line" + + assert.Equal(t, "STRIPE_KEY="+portcullis.Marker+" and the rest of the line", portcullis.Redact(in)) + matches := portcullis.Find(in) + require.Len(t, matches, 1) + assert.Equal(t, key, matches[0].Value) + }) + } +} + +// TestRedactCoversGluedStripeKeys: with no separator between two +// keys, the body of the first takes the `sk` / `rk` of the second. +// What is left of the second key must not stay behind in clear. +func TestRedactCoversGluedStripeKeys(t *testing.T) { + t.Parallel() + + for _, prefixes := range [][2]string{ + {"sk_live_", "sk_live_"}, + {"sk_live_", "rk_test_"}, + {"rk_prod_", "sk_test_"}, + } { + // 247 is the body ceiling: just under it, the first body has + // room for only part of the next prefix. + for _, bodyLen := range []int{24, 32, 99, 245, 246, 247} { + glued := prefixes[0] + strings.Repeat("a", bodyLen) + + prefixes[1] + strings.Repeat("b", bodyLen) + + prefixes[0] + strings.Repeat("c", bodyLen) + + assert.Equal(t, "key="+portcullis.Marker+" end", portcullis.Redact("key="+glued+" end"), glued) + matches := portcullis.Find(glued) + require.Len(t, matches, 1, glued) + assert.Equal(t, glued, matches[0].Value) + } + } + + // Only another key carries the match on. An environment-looking + // suffix is not one. + key := "sk_live_" + strings.Repeat("a", 24) + assert.Equal(t, portcullis.Marker+"_prod_environment", portcullis.Redact(key+"_prod_environment")) +} + // TestRedactDoesNotSwallowAdjacentTextAfterPrefixedTokens pins the // upper bounds of the rules whose bodies share a character class // with arbitrary trailing text. Without an explicit ceiling on the @@ -494,6 +628,31 @@ func TestRedactDoesNotSwallowAdjacentTextAfterPrefixedTokens(t *testing.T) { } } +// TestREADMEExample checks the inputs and results shown at the top of +// the README, the first thing a new user pastes. It once kept showing +// a GitHub token that the checksum validator had stopped accepting. +func TestREADMEExample(t *testing.T) { + t.Parallel() + + const ( + in = "Run this with AWS_ACCESS_KEY_ID=AKIARZPUZDIKQEXAMPLE please." + out = "Run this with AWS_ACCESS_KEY_ID=[REDACTED] please." + clean = "not a secret" + leaking = "AWS_ACCESS_KEY_ID=AKIARZPUZDIKQEXAMPLE" + ) + readme, err := os.ReadFile("README.md") + require.NoError(t, err) + for _, literal := range []string{in, out, clean, leaking} { + if quoted := strconv.Quote(literal); !strings.Contains(string(readme), quoted) { + t.Fatalf("README.md no longer shows %s: update this test along with it", quoted) + } + } + + assert.Equal(t, out, portcullis.Redact(in)) + assert.False(t, portcullis.Contains(clean)) + assert.True(t, portcullis.Contains(leaking)) +} + // TestContainsIgnoresHarmlessText: pure digit strings, plain English, // and the empty string must never trip detection. func TestContainsIgnoresHarmlessText(t *testing.T) { @@ -606,6 +765,36 @@ func TestMarkerIsNotASecret(t *testing.T) { portcullis.Redact("prefix "+portcullis.Marker+" suffix")) } +// TestMarkerInSecretPositionIsNotASecret covers the rules whose secret +// class is broad enough to match the marker itself, such as the +// password of a connection string. Reporting it would make every +// already-redacted connection string look like a leak. +func TestMarkerInSecretPositionIsNotASecret(t *testing.T) { + t.Parallel() + + redacted := "redis://:" + portcullis.Marker + "@cache.example.com:6379" + + assert.False(t, portcullis.Contains(redacted)) + assert.Empty(t, portcullis.Find(redacted)) + assert.Equal(t, redacted, portcullis.Redact(redacted)) + + // A real secret is still reported: one that another rule finds + // next to the marker, one that the same rule finds after it, and + // one wrapped around it. + pat := "dckr_pat_" + strings.Repeat("A", 27) + matches := portcullis.Find(redacted + " " + pat) + require.Len(t, matches, 1) + assert.Equal(t, pat, matches[0].Value) + assert.True(t, portcullis.Contains(redacted+" "+pat)) + + matches = portcullis.Find(redacted + " " + redacted + " redis://:hunter2hunter2@replica.example.com:6379") + require.Len(t, matches, 1) + assert.Equal(t, "hunter2hunter2", matches[0].Value) + assert.True(t, portcullis.Contains(redacted+" "+redacted+" redis://:hunter2hunter2@replica.example.com:6379")) + + assert.True(t, portcullis.Contains("redis://:hunter2"+portcullis.Marker+"@cache.example.com:6379")) +} + // TestRedactDetectsSecretsAcrossWordBoundaries exercises the change // that dropped the leading and trailing word-boundary anchors from // the rule expressions. Before the change, only secrets that stood diff --git a/prefilter_test.go b/prefilter_test.go index bf015c2..a4a7f5a 100644 --- a/prefilter_test.go +++ b/prefilter_test.go @@ -1,6 +1,7 @@ package portcullis import ( + "math/rand" "strings" "testing" @@ -30,26 +31,105 @@ func referenceSeparatedDigitRun(text string, minDigits int, ws bool) bool { return false } -func FuzzSeparatedDigitRun(f *testing.F) { +func referenceIBANAnchor(text string) bool { + for i := 0; i+3 < len(text); i++ { + if isASCIILetter(text[i]) && isASCIILetter(text[i+1]) && + isASCIIDigit(text[i+2]) && isASCIIDigit(text[i+3]) { + return true + } + } + return false +} + +// referenceShapes spells out each shape on its own, one pass per +// shape, as the definition [scanShapes] has to agree with. +func referenceShapes(text string) (found shapes) { + if referenceSeparatedDigitRun(text, 13, true) { + found |= shapeCard + } + if referenceSeparatedDigitRun(text, 9, false) { + found |= shapeSSN + } + if referenceIBANAnchor(text) { + found |= shapeIBAN + } + return found +} + +func FuzzScanShapes(f *testing.F) { for _, value := range []string{ "", "clean prose", "123456789", "4111 1111 1111 1111", "1\t2\n3\r4\v5\f6-7 8 9", "1 234567890123", "é123\xff456789", "12345678", "123456789012", "1234567890123", "1234- 567890123", "1234\t\n567890123", "12345678-", "123456789012\t", + "GB82", "B82", "GB8", "G8B2", "12GB82", "1 GB82", "1234\tGB82 WEST", + "123 45\t6789", "12345\t6789 12345", "DE89 3704 0044 0532 0130 00 123-45-6789", } { f.Add(value) } f.Fuzz(func(t *testing.T, value string) { - for _, n := range []int{-1, 0, 1, 9, 13, 19} { - for _, ws := range []bool{false, true} { - assert.Equal(t, referenceSeparatedDigitRun(value, n, ws), hasSeparatedDigitRun(value, n, ws), - "input %q, threshold %d, whitespace %t", value, n, ws) + assert.Equal(t, referenceShapes(value), scanShapes(value), "input %q", value) + }) +} + +// TestScanShapesMatchesReference feeds [scanShapes] digit-heavy noise, +// the inputs where its single pass has the most state to get wrong. +func TestScanShapesMatchesReference(t *testing.T) { + t.Parallel() + + const alphabet = "0123456789- \t\n\r\v\fabZ.\xc3" + rng := rand.New(rand.NewSource(1)) + for range 200_000 { + buf := make([]byte, rng.Intn(48)) + for i := range buf { + if rng.Intn(3) == 0 { + buf[i] = alphabet[rng.Intn(len(alphabet))] + } else { + buf[i] = byte('0' + rng.Intn(10)) } } - }) + value := string(buf) + if want, got := referenceShapes(value), scanShapes(value); want != got { + t.Fatalf("input %q: want %03b, got %03b", value, want, got) + } + } +} + +// TestShapesAreNecessaryForTheirRules checks the claim every shape +// rests on: an input its rule's regex matches always has the shape. +// A gate stricter than the regex would drop matches, and do so +// depending on what else the input happens to contain. +func TestShapesAreNecessaryForTheirRules(t *testing.T) { + t.Parallel() + + rs := compiledRuleSet() + // Some letters and separators, `ſ`, which case-folds to an ASCII + // letter without being one, and mostly digits. + alphabet := []string{"A", "b", "Z", "ſ", "-", ".", " ", "\t", "\n"} + for d := '0'; d <= '9'; d++ { + alphabet = append(alphabet, string(d), string(d)) + } + rng := rand.New(rand.NewSource(1)) + for range 20_000 { + var b strings.Builder + for range rng.Intn(40) { + b.WriteString(alphabet[rng.Intn(len(alphabet))]) + } + text := b.String() + found := scanShapes(text) + for i := range rs.rules { + r := &rs.rules[i] + if r.shape == 0 || found&r.shape != 0 { + continue + } + if loc := r.compile().re.FindStringIndex(text); loc != nil { + t.Fatalf("rule %d matches %q in %q, which lacks the rule's shape", i, text[loc[0]:loc[1]], text) + } + } + } } -func BenchmarkDigitPrefilter(b *testing.B) { +func BenchmarkScanShapes(b *testing.B) { for _, tc := range []struct { name, input string }{ @@ -62,8 +142,7 @@ func BenchmarkDigitPrefilter(b *testing.B) { b.Run(tc.name, func(b *testing.B) { b.ReportAllocs() for b.Loop() { - maybeCreditCard(tc.input) - maybeSSN(tc.input) + scanShapes(tc.input) } }) } diff --git a/rules.go b/rules.go index 413f473..7645899 100644 --- a/rules.go +++ b/rules.go @@ -39,12 +39,12 @@ type rule struct { keywords []string caseSensitive bool validator func(string) bool - // prefilter is a cheap necessary-condition byte scan for - // keywordless rules, which the AC pre-filter can't gate. When set, - // the rule's regex only runs on inputs the prefilter accepts; it - // must never reject an input the regex would match. Ignored for - // rules that carry keywords. - prefilter func(string) bool + // shape is a cheap necessary condition for keywordless rules, + // which the AC pre-filter can't gate. When set, the rule's regex + // only runs on inputs where [scanShapes] finds that shape; it + // must never be missing from an input the regex would match. + // Ignored for rules that carry keywords. + shape shapes } // asSecretGroup wraps a `?P…` fragment in a plain group so @@ -88,6 +88,28 @@ func contextual(vendor, body string) string { `(?P` + body + `)(?:[\x60'"\s;]|\\[nr]|$)` } +// chain matches head then body, and carries on over every further +// link and body glued to them without a separator. A body's character +// class also takes the first characters of a token that follows it +// directly (the `sk` of `sk_live_…`), so a match that ended with the +// first body would leave the rest of the second token unmatched and +// in clear. A run of glued tokens comes out as a single match. It +// ends early at a token too short to match without those borrowed +// characters, which nothing short of lookahead can tell apart. +func chain(head, link, body string) string { + return head + `(?:` + body + link + `)*` + body +} + +// stripeKey builds the expression of the Stripe secret (`sk`) and +// restricted (`rk`) key rules: `__`. Current keys +// (`sk_live_51…`) carry a 99-character body where the original format +// had 24, and a ceiling that is too low leaves the tail of a key in +// clear, so it sits at TruffleHog's 247. +func stripeKey(prefix string) string { + const env = `_(?:test|live|prod)_` + return `(?i)` + chain(prefix+env, `[sr]k`+env, `[0-9a-z]{10,247}`) +} + // rules is the source-form catalogue, kept verbatim from upstream so // future updates apply cleanly. [compiledRuleSet] resolves it into // the regex-compiled form actually used at scan time. @@ -177,8 +199,14 @@ func rules() []rule { keywords: []string{"shpss_", "shpat_", "shpca_", "shppa_"}, }, { - // slack-access-token - expression: asSecretGroup(`?Pxox[baprs]-([0-9a-zA-Z]{10,48})`), + // slack-access-token. Slack tokens are dash-separated: one to + // three numeric IDs, then the secret (`xoxb---`, `xoxp----`). + // The match has to reach that last segment, or the redaction + // leaves the secret itself in clear. It stops there too, so + // whatever is hyphenated onto a token stays out of it. The + // second alternative is the original single-segment shape. + expression: asSecretGroup(`?P` + chain(`xox[baprs]-`, `xox[baprs]-`, `(?:(?:[0-9]{8,14}-){1,3}[0-9a-zA-Z]{18,128}|[0-9a-zA-Z]{10,48})`)), keywords: []string{"xoxb-", "xoxa-", "xoxp-", "xoxr-", "xoxs-"}, }, { @@ -188,7 +216,7 @@ func rules() []rule { // intentionally NOT redacted: Stripe publishable keys are // designed to be embedded in client-side code and are not // considered secret. - expression: asSecretGroup(`?P(?i)sk_(test|live|prod)_[0-9a-z]{10,32}`), + expression: asSecretGroup(`?P` + stripeKey(`sk`)), keywords: []string{"sk_test_", "sk_live_", "sk_prod_"}, }, { @@ -675,7 +703,7 @@ func rules() []rule { // a restricted key still grants the scoped Stripe permissions // it was issued with, so it must be redacted. The `prod` env // tag was added in 2024 alongside the `test` / `live` modes. - expression: `(?i)rk_(test|live|prod)_[0-9a-z]{10,32}`, + expression: stripeKey(`rk`), keywords: []string{"rk_test_", "rk_live_", "rk_prod_"}, }, { @@ -748,20 +776,21 @@ func rules() []rule { // discord-bot-token. Three-part dotted format issued for bot // applications: `.<6-char timestamp>.<27+ // char HMAC>`. The first segment is the bot's user-ID base64 - // encoded; current Discord IDs (2018 onwards) base64 to a - // leading `MT`/`Mz`/`ND`/`NT`/`Nz`/`OD` byte pair, which we - // list as keywords so the AC pre-filter still skips inputs - // without a plausible token prefix. The structural shape - // (M-or-N-or-O-prefixed body, two literal dots, fixed segment - // widths) keeps the regex itself specific. + // encoded, so the ID's leading decimal digit (1-9) fixes its + // first two characters: `MT`/`Mj`/`Mz`/`ND`/`NT`/`Nj`/`Nz`/ + // `OD`/`OT`. The regex requires one of the nine, and we list + // them as keywords so the AC pre-filter still skips inputs + // without a plausible token prefix. Together with the two + // literal dots and the fixed segment widths, that keeps the + // rule specific. // // caseSensitive is critical here: the lower-cased forms of // these two-letter prefixes (`mt`, `nd`, `od`, …) are common // English bigrams and trip the keyword pre-filter on ~30% of // arbitrary text files. Pinning the case lets the AC pass // drop the regex run on those files. - expression: `[MNO][A-Za-z\d_-]{23,25}\.[\w-]{6,7}\.[\w-]{27,38}`, - keywords: []string{"MT", "Mz", "ND", "NT", "Nz", "OD"}, + expression: `(?:M[Tjz]|N[DTjz]|O[DT])[A-Za-z\d_-]{22,24}\.[\w-]{6,7}\.[\w-]{27,38}`, + keywords: []string{"MT", "Mj", "Mz", "ND", "NT", "Nj", "Nz", "OD", "OT"}, caseSensitive: true, }, { @@ -1961,7 +1990,7 @@ func rules() []rule { // not get redacted as cards. expression: `(?P\b(?:4\d{3}|5[1-5]\d{2}|2[2-7]\d{2}|3[47]\d{2}|6(?:011|5\d{2})|3(?:0[0-5]|[68]\d)\d)(?:[-\s]?\d){9,15}\b)`, validator: validLuhn, - prefilter: maybeCreditCard, + shape: shapeCard, }, { // iban. Mirrors memclaw's IBAN detector: broad ISO 13616 @@ -1969,9 +1998,12 @@ func rules() []rule { // would let a match run into the next line), gated by the // registered country code and length and the official mod-97 // checksum to avoid flagging arbitrary account-like strings. - expression: `(?i)(?P\b[A-Z]{2}\d{2}(?:[ ]?[A-Z0-9]{4}){2,7}(?:[ ]?[A-Z0-9]{1,3})?\b)`, + // Both cases are spelled out rather than folded with `(?i)`, + // which would also let `ſ` and the Kelvin sign in as letters, + // past the ASCII ones [shapeIBAN] looks for. + expression: `(?P\b[A-Za-z]{2}\d{2}(?:[ ]?[A-Za-z0-9]{4}){2,7}(?:[ ]?[A-Za-z0-9]{1,3})?\b)`, validator: validIBAN, - prefilter: maybeIBAN, + shape: shapeIBAN, }, { // us-ssn. Mirrors memclaw's national-ID rule for US Social @@ -1979,7 +2011,7 @@ func rules() []rule { // area / group / serial ranges (000, 666, 9xx, 00, 0000). expression: `(?P\b\d{3}[- ]?\d{2}[- ]?\d{4}\b)`, validator: validUSSSN, - prefilter: maybeSSN, + shape: shapeSSN, }, { @@ -1993,12 +2025,30 @@ func rules() []rule { } } +// compiledMatch is what [compiledRule.compile] yields: the rule's +// regex, the index of its `secret` subgroup (-1 without one) and its +// optional validator. +type compiledMatch struct { + re *regexp.Regexp + secretIdx int + validator func(string) bool +} + +// accepts reports whether value, a span matched by the rule's regex, +// is a secret: it has to pass the rule's validator, and it must not +// be the [Marker] itself, which broad classes such as a +// connection-string password match in already-redacted text. +func (c *compiledMatch) accepts(value string) bool { + return value != Marker && (c.validator == nil || c.validator(value)) +} + // compiledRule is the runtime form of a [rule]: its keywords are // folded into a [kwMask] over the catalogue's shared keyword index, -// and its regex is compiled lazily on first match. A clean input — -// the overwhelmingly common case — only runs the small set of -// keywordless validator-gated PII expressions; all keyworded secret -// rules still stay behind the AC pre-filter. +// and its regex is compiled lazily, the first time an input gets +// past the rule's gate. An input with no keyword and no PII shape — +// the overwhelmingly common case — runs no expression at all: the +// keyworded secret rules stay behind the AC pre-filter and the +// keywordless PII ones behind their [rule.shape]. // // caseSensitive, when true, requires the post-filter step (see // [compiledRule.passes]) to find at least one of [csKW] verbatim @@ -2006,32 +2056,26 @@ func rules() []rule { // already matched. This avoids running the regex on every file // that happens to contain a folded form of a CS keyword (e.g. the // English bigrams `mt`/`nd`/`od` for Discord, `sk` for Twilio). -type compiledMatch struct { - re *regexp.Regexp - secretIdx int - validator func(string) bool -} - type compiledRule struct { kwBits kwMask caseSensitive bool csKW []string - prefilter func(string) bool + shape shapes compile func() compiledMatch // memoised } // passes returns true when the rule should run its regex against -// text: keywordless rules run whenever their cheap [rule.prefilter] -// byte scan accepts the input (a necessary condition for the regex to -// match, so clean prose skips the expensive expression); otherwise +// text: keywordless rules run whenever the input has their cheap +// [rule.shape] (a necessary condition for the regex to match, so +// clean prose skips the expensive expression); otherwise // the AC pre-filter must have matched, and — for case-sensitive // rules — at least one keyword must appear in the original input // without the case-fold. The post-filter cost is one // [strings.Contains] per CS keyword, two orders of magnitude // cheaper than running the rule's regex over the whole file. -func (r *compiledRule) passes(found kwMask, text string) bool { +func (r *compiledRule) passes(found kwMask, seen *lazyShapes, text string) bool { if r.kwBits.empty() { - return r.prefilter == nil || r.prefilter(text) + return r.shape == 0 || seen.has(text, r.shape) } if !found.overlaps(r.kwBits) { return false @@ -2095,7 +2139,7 @@ var compiledRuleSet = sync.OnceValue(func() *ruleSet { kwBits: bits, caseSensitive: r.caseSensitive, csKW: r.keywords, - prefilter: r.prefilter, + shape: r.shape, compile: sync.OnceValue(func() compiledMatch { re := regexp.MustCompile(expr) return compiledMatch{re: re, secretIdx: re.SubexpIndex("secret"), validator: validator} diff --git a/validators.go b/validators.go index a7b3407..9493bf5 100644 --- a/validators.go +++ b/validators.go @@ -222,7 +222,7 @@ func validCloudflareAPIKey(token string) bool { } func validAWSBedrockLongLivedKey(token string) bool { - if len(token) != 132 || len(token)%4 != 0 { + if len(token) != 132 { return false } decoded, err := base64.StdEncoding.DecodeString(token[4:]) @@ -235,62 +235,92 @@ func validAWSBedrockLongLivedKey(token string) bool { // The keywordless PII rules (credit card, IBAN, SSN) can't hide // behind the Aho–Corasick keyword pre-filter, so without a cheap gate // their expensive regexes would run over every clean input. Each -// prefilter below is a necessary condition for its rule's regex to -// match — it may pass inputs the regex ultimately rejects, but it -// never rejects an input the regex would accept — so clean prose -// (which has no long digit runs) skips the regex entirely. - -// hasSeparatedDigitRun reports whether text contains a run of at -// least minDigits decimal digits in which consecutive digits are -// split by at most one separator byte. Separators are '-' and ' '; -// the wider ASCII-whitespace set (tab / newline / …) is folded in -// when ws is true, to mirror the card rule's [-\s] separator class. -func hasSeparatedDigitRun(text string, minDigits int, ws bool) bool { +// shape below is a necessary condition for its rule's regex to +// match — it may show up in inputs the regex ultimately rejects, but +// it is never missing from an input the regex would accept — so clean +// prose (which has no long digit runs) skips the regex entirely. +type shapes uint8 + +const ( + // shapeCard gates the card rule: its regex consumes at least 13 + // digits (a 4-digit issuer group plus 9+ more) in a run where + // consecutive digits are split by at most one byte of its [-\s] + // separator class. + shapeCard shapes = 1 << iota + // shapeSSN gates the SSN rule: its regex is 9 digits split by at + // most single '-'/' ' separators. + shapeSSN + // shapeIBAN gates the IBAN rule on its mandatory country-code + + // check-digit anchor: two ASCII letters immediately followed by + // two ASCII digits. + shapeIBAN + + allShapes = shapeCard | shapeSSN | shapeIBAN +) + +// scanShapes reports every shape text contains, in a single pass. +// They all need a digit, so prose is skipped byte by byte without +// keeping any state and only the digit runs get a closer look. +func scanShapes(text string) (found shapes) { for i := 0; i < len(text); { - // Skip prose without maintaining digit-run state for every byte. - for i < len(text) && (text[i] < '0' || text[i] > '9') { + for i < len(text) && !isASCIIDigit(text[i]) { i++ } - run := 0 - for i < len(text) && text[i] >= '0' && text[i] <= '9' { - run++ - if run >= minDigits { - return true + // text[i], if any, opens a digit run: the one place where + // letters can sit right before digits. + if i >= 2 && i+1 < len(text) && isASCIIDigit(text[i+1]) && + isASCIILetter(text[i-1]) && isASCIILetter(text[i-2]) { + found |= shapeIBAN + } + // Count the digits that at most one separator byte splits + // from the previous one. '-' and ' ' separate for both rules; + // the rest of ASCII whitespace only does for cards, so it + // starts the SSN count over. + card, ssn := 0, 0 + for i < len(text) && isASCIIDigit(text[i]) { + card++ + ssn++ + if card >= 13 { + found |= shapeCard + } + if ssn >= 9 { + found |= shapeSSN } i++ if i < len(text) { - c := text[i] - if c == '-' || c == ' ' || - (ws && (c == '\t' || c == '\n' || c == '\r' || c == '\v' || c == '\f')) { + switch text[i] { + case '-', ' ': + i++ + case '\t', '\n', '\r', '\v', '\f': + ssn = 0 i++ } } } + if found == allShapes { + break + } } - return false + return found } -// maybeCreditCard gates the card rule: its regex consumes at least 13 -// digits (a 4-digit issuer group plus 9+ more) in a single [-\s]- -// separated run. -func maybeCreditCard(text string) bool { return hasSeparatedDigitRun(text, 13, true) } - -// maybeSSN gates the SSN rule: its regex is 9 digits split by at most -// single '-'/' ' separators. -func maybeSSN(text string) bool { return hasSeparatedDigitRun(text, 9, false) } - -// maybeIBAN gates the IBAN rule on its mandatory country-code + -// check-digit anchor: two ASCII letters immediately followed by two -// ASCII digits. -func maybeIBAN(text string) bool { - for i := 0; i+3 < len(text); i++ { - if isASCIILetter(text[i]) && isASCIILetter(text[i+1]) && - text[i+2] >= '0' && text[i+2] <= '9' && - text[i+3] >= '0' && text[i+3] <= '9' { - return true - } +// lazyShapes runs [scanShapes] over an input at most once, and only +// if a keywordless rule is reached at all: [Contains] often returns +// on a keyworded rule before that. +type lazyShapes struct { + scanned bool + found shapes +} + +func (l *lazyShapes) has(text string, shape shapes) bool { + if !l.scanned { + l.found, l.scanned = scanShapes(text), true } - return false + return l.found&shape != 0 +} + +func isASCIIDigit(c byte) bool { + return c >= '0' && c <= '9' } func isASCIILetter(c byte) bool {