diff --git a/internal/redaction/redaction.go b/internal/redaction/redaction.go index 24685eca9..268af1553 100644 --- a/internal/redaction/redaction.go +++ b/internal/redaction/redaction.go @@ -7,8 +7,10 @@ import ( "reflect" "regexp" "sort" + "strconv" "strings" "unicode" + "unicode/utf8" ) const ( @@ -68,12 +70,59 @@ var sensitiveKeys = map[string]struct{}{ "zero_api_key": {}, } +// ctrlGap matches C0/C1 bytes (Cc other than tab/LF/CR, plus lone Latin-1 C1) +// between characters of a secret shape. Matching stays on the original string: +// a deleted control is never a join, so \b still treats wordchar+control as a +// boundary and tokens that were never adjacent stay that way. Tab/LF/CR are +// excluded so log line structure is unchanged. \x{FFFD} lets the regexp locate +// a lone invalid UTF-8 byte; validSecretControlGaps subsequently accepts only +// raw C1 bytes and rejects a real, valid UTF-8 U+FFFD rune. +const ctrlGap = `[\x00-\x08\x0b\x0c\x0e-\x1f\x7f\x80-\x9f\x{FFFD}]*` + +// ctrlLit quotes s as a regexp literal with ctrlGap strictly between runes, so +// a NUL/ESC/C1 may split the literal without letting a match end on a gap. +func ctrlLit(s string) string { + var b strings.Builder + b.Grow(len(s) * (1 + len(ctrlGap))) + first := true + for _, r := range s { + if !first { + b.WriteString(ctrlGap) + } + b.WriteString(regexp.QuoteMeta(string(r))) + first = false + } + return b.String() +} + +func ctrlJoin(parts ...string) string { + return strings.Join(parts, ctrlGap) +} + +// secretBody generates a regex matching at least minimum body characters, +// allowing C0/C1 control gaps between any characters. It always starts and ends +// on a class character (never on a gap). +func secretBody(class string, minimum int, unbounded bool) string { + if minimum <= 0 { + return "" + } + quantifier := strconv.Itoa(minimum - 1) + if unbounded { + return class + `(?:` + ctrlGap + class + `){` + quantifier + `,}` + } + return class + `(?:` + ctrlGap + class + `){` + quantifier + `}` +} + // openaiKeyPattern mirrors secrets.Scan's broad sk- body. Known OpenAI // prefixes (sk-proj-/sk-svcacct-/sk-admin-) are always redacted; other sk- // digit-free matches with an interior hyphen are left alone (kebab-case false // positives), while digit-free legacy sk- credentials are still redacted. // Applied via ReplaceAllStringFunc rather than the plain list below. -var openaiKeyPattern = regexp.MustCompile(`\bsk-[A-Za-z0-9_-]{20,}`) +var openaiKeyPattern = regexp.MustCompile(`\b` + ctrlJoin(ctrlLit("sk-"), secretBody(`[A-Za-z0-9_-]`, 20, true))) + +// plainOpenaiKeyPattern is the non-gap-aware counterpart of openaiKeyPattern, +// used for boundary resolution on logical (control-stripped) candidates. +var plainOpenaiKeyPattern = regexp.MustCompile(`\bsk-[A-Za-z0-9_-]{20,}`) // textSecretPatterns mirror secrets.Scan for end-boundary behavior and the // shared high-confidence shapes. A leading \b keeps each pattern from firing @@ -84,16 +133,63 @@ var openaiKeyPattern = regexp.MustCompile(`\bsk-[A-Za-z0-9_-]{20,}`) // (not in secrets.Scan); ASIA temporary access keys are kept alongside AKIA. // openai keys are handled separately (digit filter). JWT has a strict form // (both segments start with eyJ) and a looser three-segment form. -var textSecretPatterns = []*regexp.Regexp{ - regexp.MustCompile(`\bsk-ant-(?:api\d{2}-)?[A-Za-z0-9_-]{20,}`), - regexp.MustCompile(`\bgithub_pat_[A-Za-z0-9_]{22,}`), - regexp.MustCompile(`\bgh[pousr]_[A-Za-z0-9]{36,}`), - regexp.MustCompile(`\bglpat-[A-Za-z0-9_-]{12,}`), - regexp.MustCompile(`\bAIza[0-9A-Za-z\-_]{35,}`), - regexp.MustCompile(`\bxox[baprs]-[A-Za-z0-9-]{10,}`), - regexp.MustCompile(`\b(?:AKIA|ASIA)[A-Z0-9]{16}`), - regexp.MustCompile(`\beyJ[A-Za-z0-9_-]{10,}\.eyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}`), - regexp.MustCompile(`\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}`), +// ctrlGap between shape characters keeps NUL/ESC/C1 split secrets matching +// without stripping those bytes out of the subject first. +type secretShape struct { + textPattern *regexp.Regexp + plainPattern *regexp.Regexp + minLen int + requireDots bool +} + +var secretShapes = []secretShape{ + { + textPattern: regexp.MustCompile(`\b` + ctrlLit("sk-ant-") + ctrlGap + `(?:` + ctrlJoin(ctrlLit("api"), `\d`, `\d`, `-`) + ctrlGap + `)?` + secretBody(`[A-Za-z0-9_-]`, 20, true)), + plainPattern: regexp.MustCompile(`\bsk-ant-(?:api\d{2}-)?[A-Za-z0-9_-]{20,}`), + minLen: 27, // sk-ant- (7) + 20 + }, + { + textPattern: regexp.MustCompile(`\b` + ctrlJoin(ctrlLit("github_pat_"), secretBody(`[A-Za-z0-9_]`, 22, true))), + plainPattern: regexp.MustCompile(`\bgithub_pat_[A-Za-z0-9_]{22,}`), + minLen: 33, // github_pat_ (11) + 22 + }, + { + textPattern: regexp.MustCompile(`\b` + ctrlJoin(ctrlLit("gh"), `[pousr]`, `_`, secretBody(`[A-Za-z0-9]`, 36, true))), + plainPattern: regexp.MustCompile(`\bgh[pousr]_[A-Za-z0-9]{36,}`), + minLen: 40, // gh[pousr]_ (4) + 36 + }, + { + textPattern: regexp.MustCompile(`\b` + ctrlJoin(ctrlLit("glpat-"), secretBody(`[A-Za-z0-9_-]`, 20, true))), + plainPattern: regexp.MustCompile(`\bglpat-[A-Za-z0-9_-]{20,}`), + minLen: 26, // glpat- (6) + 20 + }, + { + textPattern: regexp.MustCompile(`\b` + ctrlJoin(ctrlLit("AIza"), secretBody(`[0-9A-Za-z\-_]`, 35, true))), + plainPattern: regexp.MustCompile(`\bAIza[0-9A-Za-z\-_]{35,}`), + minLen: 39, // AIza (4) + 35 + }, + { + textPattern: regexp.MustCompile(`\b` + ctrlJoin(ctrlLit("xox"), `[baprs]`, `-`, secretBody(`[A-Za-z0-9-]`, 10, true))), + plainPattern: regexp.MustCompile(`\bxox[baprs]-[A-Za-z0-9-]{10,}`), + minLen: 15, // xox[baprs]- (5) + 10 + }, + { + textPattern: regexp.MustCompile(`\b` + ctrlJoin(`(?:`+ctrlLit("AKIA")+`|`+ctrlLit("ASIA")+`)`, secretBody(`[A-Z0-9]`, 16, false))), + plainPattern: regexp.MustCompile(`\b(?:AKIA|ASIA)[A-Z0-9]{16}`), + minLen: 20, // AKIA/ASIA (4) + 16 + }, + { + textPattern: regexp.MustCompile(`\b` + ctrlJoin(ctrlLit("eyJ"), secretBody(`[A-Za-z0-9_-]`, 10, true), `\.`, ctrlLit("eyJ"), secretBody(`[A-Za-z0-9_-]`, 10, true), `\.`, secretBody(`[A-Za-z0-9_-]`, 10, true))), + plainPattern: regexp.MustCompile(`\beyJ[A-Za-z0-9_-]{10,}\.eyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}`), + minLen: 38, // JWT (3 + 10 + 1 + 3 + 10 + 1 + 10) + requireDots: true, + }, + { + textPattern: regexp.MustCompile(`\b` + ctrlJoin(ctrlLit("eyJ"), secretBody(`[A-Za-z0-9_-]`, 10, true), `\.`, secretBody(`[A-Za-z0-9_-]`, 10, true), `\.`, secretBody(`[A-Za-z0-9_-]`, 10, true))), + plainPattern: regexp.MustCompile(`\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}`), + minLen: 34, // JWT (3 + 10 + 1 + 10 + 1 + 10) + requireDots: true, + }, } var ( @@ -172,6 +268,9 @@ func keyLooksSensitive(normalized string) bool { func RedactString(value string, options Options) string { replacement := replacement(options) + // Match on the original string. Shape patterns allow C0/C1 gaps between + // characters so a split secret still matches; stripping first would join + // tokens that were never adjacent and make \b miss a leading wordchar. redacted := value if len(options.ExtraSecretValues) > 0 { secrets := append([]string{}, options.ExtraSecretValues...) @@ -224,19 +323,456 @@ func RedactString(value string, options Options) string { } return parts[1] + parts[2] + "=" + replacement }) - // openai keys first so the filter can drop kebab-case false positives - // before any other pattern rewrites nearby text. - redacted = openaiKeyPattern.ReplaceAllStringFunc(redacted, func(match string) string { - if !knownOpenAIKeyPrefix(match) && !secretMatchHasDigit(match) && - strings.Contains(strings.TrimPrefix(match, "sk-"), "-") { - return match + // Match high-confidence specialized shapes first. In particular, the broad + // sk- pattern may reach its minimum before a control inside a longer + // Anthropic key; letting the Anthropic shape consume that split first avoids + // leaving a recognizable credential suffix behind. + var allSpans []span + for _, shape := range secretShapes { + allSpans = append(allSpans, findSpansForShape(redacted, shape, false)...) + } + openaiShape := secretShape{ + textPattern: openaiKeyPattern, + plainPattern: plainOpenaiKeyPattern, + minLen: minOpenAILen, + requireDots: false, + } + allSpans = append(allSpans, findSpansForShape(redacted, openaiShape, true)...) + redacted = applySpans(redacted, allSpans, replacement) + return redacted +} + +const minOpenAILen = 23 // sk- (3) + 20 + +type controlSpan struct { + start int + end int + validGap bool +} + +type logicalCandidate struct { + logical string + origEnds []int + spans []controlSpan +} + +func extractLogicalCandidate(s string) logicalCandidate { + var logical strings.Builder + logical.Grow(len(s)) + var origEnds []int + origEnds = make([]int, 0, len(s)) + var spans []controlSpan + + for i := 0; i < len(s); { + c := s[i] + if c < 0x80 { + if c != '\t' && c != '\n' && c != '\r' && (c < 0x20 || c == 0x7F) { + start := i + for i < len(s) && s[i] < 0x80 && s[i] != '\t' && s[i] != '\n' && s[i] != '\r' && (s[i] < 0x20 || s[i] == 0x7F) { + i++ + } + spans = append(spans, controlSpan{start: start, end: i, validGap: true}) + continue + } + logical.WriteByte(c) + i++ + origEnds = append(origEnds, i) + continue } - return replacement + if c >= 0x80 && c <= 0x9F { + start := i + for i < len(s) && s[i] >= 0x80 && s[i] <= 0x9F { + i++ + } + spans = append(spans, controlSpan{start: start, end: i, validGap: true}) + continue + } + r, size := utf8.DecodeRuneInString(s[i:]) + if r == utf8.RuneError { + start := i + i += size + spans = append(spans, controlSpan{start: start, end: i, validGap: false}) + continue + } + if unicode.IsControl(r) && r != '\t' && r != '\n' && r != '\r' { + start := i + i += size + for i < len(s) { + nr, nsize := utf8.DecodeRuneInString(s[i:]) + if unicode.IsControl(nr) && nr != '\t' && nr != '\n' && nr != '\r' { + i += nsize + } else { + break + } + } + spans = append(spans, controlSpan{start: start, end: i, validGap: true}) + continue + } + logical.WriteRune(r) + i += size + runeLen := len(string(r)) + for b := 0; b < runeLen; b++ { + origEnds = append(origEnds, i) + } + } + return logicalCandidate{ + logical: logical.String(), + origEnds: origEnds, + spans: spans, + } +} + +type span struct { + start int + end int +} + +func isWordByte(c byte) bool { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '_' +} + +var ( + anchoredAnthropicKeyPattern = regexp.MustCompile(`^sk-ant-(?:api\d{2}-)?[A-Za-z0-9_-]{20,}`) + anchoredGitHubPatPattern = regexp.MustCompile(`^github_pat_[A-Za-z0-9_]{22,}`) + anchoredGitHubClassicPattern = regexp.MustCompile(`^gh[pousr]_[A-Za-z0-9]{36,}`) + anchoredGitLabPatPattern = regexp.MustCompile(`^glpat-[A-Za-z0-9_-]{20,}`) + anchoredGoogleApiKeyPattern = regexp.MustCompile(`^AIza[0-9A-Za-z\-_]{35,}`) + anchoredSlackTokenPattern = regexp.MustCompile(`^xox[baprs]-[A-Za-z0-9-]{10,}`) + anchoredAwsKeyPattern = regexp.MustCompile(`^(?:AKIA|ASIA)[A-Z0-9]{16}`) + anchoredJwtStrictPattern = regexp.MustCompile(`^eyJ[A-Za-z0-9_-]{10,}\.eyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}`) + anchoredJwtLoosePattern = regexp.MustCompile(`^eyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}`) + anchoredOpenaiKeyPattern = regexp.MustCompile(`^sk-[A-Za-z0-9_-]{20,}`) +) + +func startsIndependentCredential(s string, allowIncompletePrefix bool) bool { + if len(s) < 15 { + return false + } + tokenEnd := len(s) + for i := 0; i < len(s); i++ { + c := s[i] + if c < 0x20 || c == 0x7F || c >= 0x80 || c == '/' || c == '\\' || c == ' ' { + tokenEnd = i + break + } + } + token := s[:tokenEnd] + if len(token) >= 15 { + if strings.HasPrefix(token, "sk-ant-") && anchoredAnthropicKeyPattern.MatchString(token) { + return true + } + if strings.HasPrefix(token, "github_pat_") && anchoredGitHubPatPattern.MatchString(token) { + return true + } + if len(token) >= 4 && token[0] == 'g' && token[1] == 'h' && (token[2] == 'p' || token[2] == 'o' || token[2] == 'u' || token[2] == 's' || token[2] == 'r') && token[3] == '_' && anchoredGitHubClassicPattern.MatchString(token) { + return true + } + if strings.HasPrefix(token, "glpat-") && anchoredGitLabPatPattern.MatchString(token) { + return true + } + if strings.HasPrefix(token, "AIza") && anchoredGoogleApiKeyPattern.MatchString(token) { + return true + } + if len(token) >= 5 && strings.HasPrefix(token, "xox") && (token[3] == 'b' || token[3] == 'a' || token[3] == 'p' || token[3] == 'r' || token[3] == 's') && token[4] == '-' && anchoredSlackTokenPattern.MatchString(token) { + return true + } + if (strings.HasPrefix(token, "AKIA") || strings.HasPrefix(token, "ASIA")) && anchoredAwsKeyPattern.MatchString(token) { + return true + } + if strings.HasPrefix(token, "eyJ") && (anchoredJwtStrictPattern.MatchString(token) || anchoredJwtLoosePattern.MatchString(token)) { + return true + } + if strings.HasPrefix(token, "sk-") { + loc := anchoredOpenaiKeyPattern.FindStringIndex(token) + if loc != nil { + candStr := token[:loc[1]] + if knownOpenAIKeyPrefix(candStr) || secretMatchHasDigit(candStr) || !strings.Contains(strings.TrimPrefix(candStr, "sk-"), "-") { + return true + } + } + } + } + if allowIncompletePrefix && (strings.HasPrefix(s, "sk-proj-") || strings.HasPrefix(s, "sk-ant-") || strings.HasPrefix(s, "github_pat_") || + strings.HasPrefix(s, "glpat-") || strings.HasPrefix(s, "AIza")) { + return true + } + return false +} + +func isCandidateValid(logPre string, shape secretShape, isOpenAI bool, dots, digits int, hasInteriorHyphen bool) bool { + if len(logPre) < shape.minLen { + return false + } + if shape.requireDots && dots < 2 { + return false + } + if isOpenAI { + if !strings.HasPrefix(logPre, "sk-") { + return false + } + if knownOpenAIKeyPrefix(logPre) { + return true + } + if digits > 0 { + return true + } + if hasInteriorHyphen { + return false + } + return true + } + if shape.plainPattern != nil && !shape.plainPattern.MatchString(logPre) { + return false + } + return true +} + +func extractSpansFromMatch(src string, matchStart, matchEnd int, shape secretShape, isOpenAI bool) ([]span, int) { + match := src[matchStart:matchEnd] + cand := extractLogicalCandidate(match) + if len(cand.logical) == 0 { + return nil, 0 + } + + var spans []span + candStartOrig := 0 + candStartLog := 0 + logCursor := 0 + runningDots := 0 + runningDigits := 0 + hasInteriorHyphen := false + lastConsumedEnd := 0 + + checkCandidate := func(logEnd int) bool { + if logEnd <= candStartLog { + return false + } + logPre := cand.logical[candStartLog:logEnd] + valid := isCandidateValid(logPre, shape, isOpenAI, runningDots, runningDigits, hasInteriorHyphen) + if valid { + origEnd := cand.origEnds[logEnd-1] + spans = append(spans, span{ + start: matchStart + candStartOrig, + end: matchStart + origEnd, + }) + lastConsumedEnd = origEnd + return true + } + return false + } + + for i := 0; i < len(cand.spans); i++ { + cSpan := cand.spans[i] + // Mixed C0, raw C1, and UTF-8 C1 spans form one gap. Classify the + // following token only after the whole run, preserving its original bytes. + for cSpan.validGap && i+1 < len(cand.spans) && cand.spans[i+1].validGap && cand.spans[i+1].start == cSpan.end { + i++ + cSpan.end = cand.spans[i].end + } + if cSpan.start < candStartOrig { + continue + } + for logCursor < len(cand.origEnds) && cand.origEnds[logCursor] <= cSpan.start { + c := cand.logical[logCursor] + if shape.requireDots && c == '.' { + runningDots++ + } + if isOpenAI { + if c >= '0' && c <= '9' { + runningDigits++ + } + if c == '-' && (logCursor-candStartLog) >= 3 { + hasInteriorHyphen = true + } + } + logCursor++ + } + + logPre := cand.logical[candStartLog:logCursor] + prefixValid := isCandidateValid(logPre, shape, isOpenAI, runningDots, runningDigits, hasInteriorHyphen) + tailInSrc := src[matchStart+cSpan.end:] + tailWindow := tailInSrc + if len(tailWindow) > 64 { + tailWindow = tailWindow[:64] + } + // A complete neighboring JWT may need more than 64 bytes to reach + // its signature. Inspect its full first token; the helper stops at + // the next delimiter, so successive gaps examine disjoint segments. + // An incomplete prefix can separate prose from a later split secret, + // but cannot end an already-valid candidate and expose its suffix. + startsNew := startsIndependentCredential(tailInSrc, !prefixValid) + if startsNew { + checkCandidate(logCursor) + candStartOrig = cSpan.end + candStartLog = logCursor + runningDots = 0 + runningDigits = 0 + hasInteriorHyphen = false + lastConsumedEnd = cSpan.end + continue + } + + if prefixValid { + if idx := strings.IndexAny(tailWindow, "/\\"); idx >= 0 { + segment := tailWindow[:idx] + if len(segment) < 15 && !strings.Contains(segment, "\n") { + if !isOpenAI || !strings.Contains(strings.TrimPrefix(logPre, "sk-"), "-") || knownOpenAIKeyPrefix(logPre) || runningDigits > 0 { + checkCandidate(logCursor) + } + if lastConsumedEnd <= 0 { + lastConsumedEnd = cSpan.start + } + return spans, lastConsumedEnd + } + } + firstWord := tailWindow + if spaceIdx := strings.IndexAny(firstWord, " \t\n\r"); spaceIdx >= 0 { + firstWord = firstWord[:spaceIdx] + } + if isOpenAI && strings.Contains(firstWord, "-") && !secretMatchHasDigit(firstWord) && !startsIndependentCredential(firstWord, true) { + checkCandidate(logCursor) + if lastConsumedEnd <= 0 { + lastConsumedEnd = cSpan.start + } + return spans, lastConsumedEnd + } + } else if idx := strings.IndexAny(tailWindow, "/\\"); idx >= 0 { + segment := tailWindow[:idx] + if len(segment) < 15 && !strings.Contains(segment, "\n") { + if lastConsumedEnd <= 0 { + lastConsumedEnd = cSpan.start + } + return spans, lastConsumedEnd + } + } + + if !cSpan.validGap { + checkCandidate(logCursor) + candStartOrig = cSpan.end + candStartLog = logCursor + runningDots = 0 + runningDigits = 0 + hasInteriorHyphen = false + lastConsumedEnd = cSpan.end + continue + } + } + + for logCursor < len(cand.origEnds) { + c := cand.logical[logCursor] + if shape.requireDots && c == '.' { + runningDots++ + } + if isOpenAI { + if c >= '0' && c <= '9' { + runningDigits++ + } + if c == '-' && (logCursor-candStartLog) >= 3 { + hasInteriorHyphen = true + } + } + logCursor++ + } + if checkCandidate(logCursor) { + lastConsumedEnd = len(match) + } + + if lastConsumedEnd <= 0 { + lastConsumedEnd = candStartOrig + } + if lastConsumedEnd <= 0 { + lastConsumedEnd = len(match) + } + return spans, lastConsumedEnd +} + +func findSpansForShape(src string, shape secretShape, isOpenAI bool) []span { + var spans []span + lastIndex := 0 + for lastIndex < len(src) { + loc := shape.textPattern.FindStringIndex(src[lastIndex:]) + if loc == nil { + break + } + matchStart := lastIndex + loc[0] + matchEnd := lastIndex + loc[1] + + if matchStart > 0 && isWordByte(src[matchStart-1]) { + // If matchStart directly abuts the end of an already matched span, + // allow it so adjacent credentials like AKIA...AKIA... both match. + abuts := false + for _, s := range spans { + if s.end == matchStart { + abuts = true + break + } + } + if !abuts { + lastIndex = matchStart + 1 + continue + } + } + + matchSpans, consumedEnd := extractSpansFromMatch(src, matchStart, matchEnd, shape, isOpenAI) + spans = append(spans, matchSpans...) + + if consumedEnd > 0 { + lastIndex = matchStart + consumedEnd + } else if matchEnd > matchStart { + lastIndex = matchEnd + } else { + lastIndex = matchStart + 1 + } + } + return spans +} + +func mergeSpans(spans []span) []span { + if len(spans) <= 1 { + return spans + } + sort.Slice(spans, func(i, j int) bool { + if spans[i].start != spans[j].start { + return spans[i].start < spans[j].start + } + return spans[i].end > spans[j].end }) - for _, pattern := range textSecretPatterns { - redacted = pattern.ReplaceAllString(redacted, replacement) + merged := make([]span, 0, len(spans)) + cur := spans[0] + for i := 1; i < len(spans); i++ { + s := spans[i] + if s.start < cur.end { + if s.end > cur.end { + cur.end = s.end + } + } else { + merged = append(merged, cur) + cur = s + } } - return redacted + merged = append(merged, cur) + return merged +} + +func applySpans(src string, spans []span, replacement string) string { + if len(spans) == 0 { + return src + } + merged := mergeSpans(spans) + var b strings.Builder + b.Grow(len(src)) + lastIndex := 0 + for _, s := range merged { + if s.start > lastIndex { + b.WriteString(src[lastIndex:s.start]) + } + b.WriteString(replacement) + lastIndex = s.end + } + if lastIndex < len(src) { + b.WriteString(src[lastIndex:]) + } + return b.String() } // knownOpenAIKeyPrefix is the redaction-side twin of secrets.knownOpenAIKeyPrefix: diff --git a/internal/redaction/redaction_test.go b/internal/redaction/redaction_test.go index 3ae228c8d..63829610c 100644 --- a/internal/redaction/redaction_test.go +++ b/internal/redaction/redaction_test.go @@ -141,3 +141,125 @@ func containsCircular(v any) bool { } return false } + +func TestRedactStringCatchesSecretsSplitByControlBytes(t *testing.T) { + // Unsplit passing is not coverage: a NUL/ESC/C1 in the body splits the + // shape so the patterns miss it unless matching allows those controls as + // gaps between body characters (without joining unrelated tokens). + const prefix = "sk-ant-api03-" + const body = "abcdefghijklmnopqrstuvwxyz" + unsplit := prefix + body + if got := RedactString(unsplit, Options{}); strings.Contains(got, body) { + t.Fatalf("unsplit secret not redacted (test setup): %q", got) + } + + cases := []struct { + name string + split string + }{ + {name: "NUL", split: "\x00"}, + {name: "ESC", split: "\x1b"}, + {name: "C1", split: "\x9b"}, + {name: "UTF-8 C1", split: string(rune(0x9B))}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + inputs := []struct { + placement string + input string + }{ + {placement: "prefix-body boundary", input: prefix + tc.split + body}, + {placement: "inside body", input: prefix + body[:13] + tc.split + body[13:]}, + } + for _, input := range inputs { + t.Run(input.placement, func(t *testing.T) { + got := RedactString(input.input, Options{}) + if got != RedactedSecret { + t.Fatalf("expected %q after %s %s split, got %q", RedactedSecret, tc.name, input.placement, got) + } + }) + } + }) + } +} + +func TestRedactStringPreservesControlAfterCredential(t *testing.T) { + const secret = "sk-ant-api03-abcdefghijklmnopqrstuvwxyz" + input := "key=" + secret + "\x00path/one.go\x00path/two.go" + want := "key=" + RedactedSecret + "\x00path/one.go\x00path/two.go" + if got := RedactString(input, Options{}); got != want { + t.Fatalf("terminal credential separator changed:\n got=%q\nwant=%q", got, want) + } +} + +func TestRedactStringSuffixCannotDisableOpenAIKeyMatch(t *testing.T) { + const secret = "sk-aaaaaaaaaaaaaaaaaaaabcdefgh" + input := "key " + secret + "\x1bkebab-case tail" + want := "key " + RedactedSecret + "\x1bkebab-case tail" + if got := RedactString(input, Options{}); got != want { + t.Fatalf("suffix changed OpenAI key classification:\n got=%q\nwant=%q", got, want) + } +} + +func TestRedactStringDistinguishesInvalidC1FromValidReplacementRune(t *testing.T) { + const prefix = "sk-ant-api03-" + const body = "abcdefghijklmnopqrstuvwxyz" + + invalidC1 := prefix + body[:13] + "\x9b" + body[13:] + if got := RedactString(invalidC1, Options{}); got != RedactedSecret { + t.Fatalf("raw invalid C1 split was not redacted: %q", got) + } + + validReplacement := prefix + body[:13] + "\uFFFD" + body[13:] + got := RedactString(validReplacement, Options{}) + if !strings.Contains(got, "\uFFFD"+body[13:]) { + t.Fatalf("valid U+FFFD was treated as a control gap: %q", got) + } +} + +func TestRedactStringPreservesAllowedWhitespaceAndUTF8(t *testing.T) { + input := "safe\tline\nnext\rfinal café" + if got := RedactString(input, Options{}); got != input { + t.Fatalf("unexpected normalization: %q", got) + } +} + +func TestRedactStringWordcharBeforeNULAnthropicKey(t *testing.T) { + // Matching on a control-stripped copy joins "id42" and the key, so \b in + // textSecretPatterns misses and the secret leaks. Matching on the original + // treats the NUL as a boundary; leaked must be false. + const secret = "sk-ant-api03-abcdefghijklmnopqrstuvwxyz" + if got := RedactString(secret, Options{}); strings.Contains(got, "sk-ant-api03-") { + t.Fatalf("unsplit secret not redacted (test setup): %q", got) + } + input := "id42\x00" + secret + got := RedactString(input, Options{}) + leaked := strings.Contains(got, secret) || strings.Contains(got, "sk-ant-api03-") + if leaked { + t.Fatalf("wordchar-before-NUL+anthropic-key leaked=true out=%q", got) + } + if !strings.Contains(got, RedactedSecret) { + t.Fatalf("wordchar-before-NUL+anthropic-key leaked=false want %q, got %q", RedactedSecret, got) + } +} + +func TestRedactStringControlBytesWithoutSecretStayIdentical(t *testing.T) { + // scrubResultSecrets sets Result.Redacted when RedactString's result != + // Output. Stripping is matching-time only: no-secret control bytes must + // remain byte-identical so Redacted stays false. + cases := []struct { + name string + input string + }{ + {name: "form feed in source", input: "package main\n\ffunc main() {}\n"}, + {name: "Windows-1252 quotes", input: "Don\x92t \x93quote\x94 me\n"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := RedactString(tc.input, Options{}) + if got != tc.input { + t.Fatalf("no-secret input not byte-identical:\n in=%q\nout=%q", tc.input, got) + } + }) + } +} diff --git a/internal/redaction/split_harness_test.go b/internal/redaction/split_harness_test.go new file mode 100644 index 000000000..ba854a0ff --- /dev/null +++ b/internal/redaction/split_harness_test.go @@ -0,0 +1,507 @@ +package redaction + +import ( + "strconv" + "strings" + "testing" +) + +// Construct deliberately synthetic tokens for shape matching without storing +// credential-shaped Slack literals that GitHub push protection rejects. +func syntheticSlackToken(digit, letter byte) string { + return "xoxb-" + strings.Repeat(string(digit), 12) + "-" + strings.Repeat(string(letter), 15) +} + +func TestSplitRedactionHarness(t *testing.T) { + // Representative secrets for all supported shapes + secrets := []struct { + name string + secret string + }{ + {"Anthropic", "sk-ant-api03-abcdefghijklmnopqrstuvwxyz1234"}, // gitleaks:allow -- synthetic redaction fixture + {"OpenAI standard", "sk-abcdefghijklmnopqrstuvwxyz12345678"}, // gitleaks:allow -- synthetic redaction fixture + {"OpenAI with hyphen and digit", "sk-aaaaaaaaaa-bbbbbbbbb1234567890"}, // gitleaks:allow -- synthetic redaction fixture + {"OpenAI proj", "sk-proj-abcdefghijklmnopqrstuvwxyz12345"}, // gitleaks:allow -- synthetic redaction fixture + {"GitHub PAT", "github_pat_11AAAAAAA0123456789abcdefghijklmnopqrstuvwxyz"}, // gitleaks:allow -- synthetic redaction fixture + {"GitHub Fine-Grained", "ghp_123456789012345678901234567890123456"}, // gitleaks:allow -- synthetic redaction fixture + {"GitLab PAT", "glpat-12345678901234567890"}, // gitleaks:allow -- synthetic redaction fixture + {"Google API", "AIzaSyD-1234567890123456789012345678901"}, // gitleaks:allow -- synthetic redaction fixture + {"Slack bot", syntheticSlackToken('1', 'a')}, + {"AWS AKIA", "AKIAIOSFODNN7EXAMPLE"}, // gitleaks:allow -- synthetic redaction fixture + {"JWT", "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ.SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c"}, // gitleaks:allow -- synthetic redaction fixture + } + + controls := []struct { + name string + char string + }{ + {"NUL", "\x00"}, + {"ESC", "\x1b"}, + {"lone C1", "\x9b"}, + {"UTF-8 C1", "\u009b"}, + } + + for _, s := range secrets { + t.Run(s.name, func(t *testing.T) { + // First verify unsplit redacts + gotUnsplit := RedactString(s.secret, Options{}) + if strings.Contains(gotUnsplit, s.secret) || !strings.Contains(gotUnsplit, RedactedSecret) { + t.Fatalf("unsplit secret %q failed to redact: %q", s.secret, gotUnsplit) + } + + // Test split at all interior positions throughout the secret + for _, ctrl := range controls { + for pos := 1; pos < len(s.secret); pos++ { + splitSecret := s.secret[:pos] + ctrl.char + s.secret[pos:] + got := RedactString(splitSecret, Options{}) + + if got != RedactedSecret { + t.Fatalf("split at pos %d with %s did not equal RedactedSecret: got %q, want %q", pos, ctrl.name, got, RedactedSecret) + } + } + } + }) + } +} + +func TestSplitRedactionMultiControlCases(t *testing.T) { + t.Run("Internal gap before minimum then terminal delimiter", func(t *testing.T) { + input := "sk-ant-api03-\x00abcdefghijklmnopqrstuvwxyz\x00path/file.go" // gitleaks:allow -- synthetic redaction fixture + got := RedactString(input, Options{}) + want := RedactedSecret + "\x00path/file.go" + if got != want { + t.Fatalf("multi-control anthropic mismatch:\n got=%q\nwant=%q", got, want) + } + }) + + t.Run("OpenAI internal gap then terminal delimiter before kebab suffix", func(t *testing.T) { + input := "sk-\x00abcdefghijklmnopqrstuv\x1bkebab-case tail" // gitleaks:allow -- synthetic redaction fixture + got := RedactString(input, Options{}) + want := RedactedSecret + "\x1bkebab-case tail" + if got != want { + t.Fatalf("multi-control openai mismatch:\n got=%q\nwant=%q", got, want) + } + }) + + t.Run("OpenAI internal gap before digit suffix then terminal delimiter", func(t *testing.T) { + input := "sk-aaaaaaaaaa-bbbbbbbbb\x001234567890\x00path/one.go" // gitleaks:allow -- synthetic redaction fixture + got := RedactString(input, Options{}) + want := RedactedSecret + "\x00path/one.go" + if got != want { + t.Fatalf("multi-control openai with digits mismatch:\n got=%q\nwant=%q", got, want) + } + }) + + t.Run("JWT multiple internal gaps and terminal delimiter", func(t *testing.T) { + input := "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9\x00.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ\x1b.SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c\x00trailing/text" // gitleaks:allow -- synthetic redaction fixture + got := RedactString(input, Options{}) + want := RedactedSecret + "\x00trailing/text" + if got != want { + t.Fatalf("multi-control jwt mismatch:\n got=%q\nwant=%q", got, want) + } + }) + + t.Run("Multiple internal controls in credential body", func(t *testing.T) { + input := "sk-ant-\x00api03-\x1babcdefghijklmnopqrstuvwxyz" // gitleaks:allow -- synthetic redaction fixture + got := RedactString(input, Options{}) + if got != RedactedSecret { + t.Fatalf("multiple internal gaps in anthropic key mismatch: got %q, want %q", got, RedactedSecret) + } + }) + + t.Run("Terminal delimiter separating two credentials", func(t *testing.T) { + input := "sk-ant-api03-abcdefghijklmnopqrstuvwxyz\x00ghp_123456789012345678901234567890123456" // gitleaks:allow -- synthetic redaction fixture + got := RedactString(input, Options{}) + want := RedactedSecret + "\x00" + RedactedSecret + if got != want { + t.Fatalf("two credentials separated by delimiter mismatch: got %q, want %q", got, want) + } + }) + + t.Run("Two same-shape keys separated by control bytes", func(t *testing.T) { + keyPairs := []struct { + name string + key1 string + key2 string + }{ + {"OpenAI", "sk-aaaaaaaaaaaaaaaaaaaabcdefgh", "sk-bbbbbbbbbbbbbbbbbbbbcdefghi"}, // gitleaks:allow -- synthetic redaction fixture + {"GitHub Fine-Grained", "ghp_123456789012345678901234567890123456", "ghp_abcdefghijklmnopqrstuvwxyz1234567890"}, // gitleaks:allow -- synthetic redaction fixture + {"GitHub PAT", "github_pat_11AAAAAAA0123456789abcdefghijklmnopqrstuvwxyz", "github_pat_22BBBBBBB0123456789abcdefghijklmnopqrstuvwxyz"}, // gitleaks:allow -- synthetic redaction fixture + {"GitLab PAT", "glpat-12345678901234567890", "glpat-abcdefghijklmnopqrst"}, // gitleaks:allow -- synthetic redaction fixture + {"Google API", "AIzaSyD-1234567890123456789012345678901", "AIzaSyD-abcdefghijklmnopqrstuvwxyz12345"}, // gitleaks:allow -- synthetic redaction fixture + {"Slack", syntheticSlackToken('1', 'a'), syntheticSlackToken('2', 'b')}, + } + ctrls := []string{"\x00", "\x1b", "\x9b", "\u009b"} + for _, pair := range keyPairs { + for _, ctrl := range ctrls { + input := pair.key1 + ctrl + pair.key2 + got := RedactString(input, Options{}) + want := RedactedSecret + ctrl + RedactedSecret + if got != want { + t.Fatalf("two %s keys separated by %q mismatch: got %q, want %q", pair.name, ctrl, got, want) + } + } + } + }) + + t.Run("Three same-shape keys separated by control bytes", func(t *testing.T) { + input := "sk-aaaaaaaaaaaaaaaaaaaabcdefgh\x00sk-bbbbbbbbbbbbbbbbbbbbcdefghi\x1bsk-ccccccccccccccccccccdefghij" // gitleaks:allow -- synthetic redaction fixture + got := RedactString(input, Options{}) + want := RedactedSecret + "\x00" + RedactedSecret + "\x1b" + RedactedSecret + if got != want { + t.Fatalf("three OpenAI keys mismatch: got %q, want %q", got, want) + } + }) + + t.Run("Short sk- token before credential", func(t *testing.T) { + input := "sk-ab\x00sk-aaaaaaaaaaaaaaaaaaaabcdefgh" // gitleaks:allow -- synthetic redaction fixture + got := RedactString(input, Options{}) + want := "sk-ab\x00" + RedactedSecret // gitleaks:allow -- synthetic redaction fixture + if got != want { + t.Fatalf("short sk- token before credential mismatch: got %q, want %q", got, want) + } + }) + + t.Run("OpenAI kebab false positive before path with digit", func(t *testing.T) { + input := "sk-my-awesome-kebab-project\x00v2/file.go" // gitleaks:allow -- synthetic redaction fixture + got := RedactString(input, Options{}) + want := "sk-my-awesome-kebab-project\x00v2/file.go" // gitleaks:allow -- synthetic redaction fixture + if got != want { + t.Fatalf("kebab project before path with digit mismatch: got %q, want %q", got, want) + } + }) + + t.Run("Complete credential followed by invalid bytes", func(t *testing.T) { + cases := []struct { + name string + suffix string + }{ + {"valid U+FFFD", "\uFFFDsuffix"}, + {"malformed byte 0xFF", "\xffsuffix"}, + {"malformed byte 0xC0", "\xc0suffix"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + input := "sk-ant-api03-abcdefghijklmnopqrstuvwxyz" + tc.suffix // gitleaks:allow -- synthetic redaction fixture + got := RedactString(input, Options{}) + want := RedactedSecret + tc.suffix + if got != want { + t.Fatalf("suffix %s mismatch: got %q, want %q", tc.name, got, want) + } + }) + } + }) +} + +func TestSplitRedactionNegativeCases(t *testing.T) { + controls := []string{"\x00", "\x1b", "\x9b", "\u009b"} + + t.Run("OpenAI kebab false positive with control", func(t *testing.T) { + kebab := "sk-my-awesome-kebab-project" // gitleaks:allow -- synthetic redaction fixture + for _, ctrl := range controls { + input := kebab[:10] + ctrl + kebab[10:] + got := RedactString(input, Options{}) + if got != input { + t.Fatalf("digit-free kebab falsely redacted with control %q: got %q, want %q", ctrl, got, input) + } + } + }) + + t.Run("Control immediately before complete credential", func(t *testing.T) { + secret := "sk-ant-api03-abcdefghijklmnopqrstuvwxyz" // gitleaks:allow -- synthetic redaction fixture + for _, ctrl := range controls { + input := "prefix" + ctrl + secret + got := RedactString(input, Options{}) + want := "prefix" + ctrl + RedactedSecret + if got != want { + t.Fatalf("control before secret mutated boundary: got %q, want %q", got, want) + } + } + }) + + t.Run("Control immediately after complete credential", func(t *testing.T) { + secret := "sk-ant-api03-abcdefghijklmnopqrstuvwxyz" // gitleaks:allow -- synthetic redaction fixture + for _, ctrl := range controls { + input := secret + ctrl + "path/file" + got := RedactString(input, Options{}) + want := RedactedSecret + ctrl + "path/file" + if got != want { + t.Fatalf("control after secret mutated delimiter: got %q, want %q", got, want) + } + } + }) + + t.Run("Kebab project before OpenAI key separated by control", func(t *testing.T) { + // Finding 1: kebab before OpenAI key separated by control + input := "sk-my-awesome-kebab-project\x00sk-abcdefghijklmnopqrstuv123456" // gitleaks:allow -- synthetic redaction fixture + got := RedactString(input, Options{}) + want := "sk-my-awesome-kebab-project\x00" + RedactedSecret // gitleaks:allow -- synthetic redaction fixture + if got != want { + t.Fatalf("kebab before key mismatch: got %q, want %q", got, want) + } + }) + + t.Run("Neighboring complete JWTs separated by control", func(t *testing.T) { + // Finding 2: neighboring complete JWTs separated by control + jwt1 := "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ.SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c" // gitleaks:allow -- synthetic redaction fixture + jwt2 := "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.4peTcaNQZNs4FcW3Usagee0" // gitleaks:allow -- synthetic redaction fixture + for _, ctrl := range controls { + input := jwt1 + ctrl + jwt2 + got := RedactString(input, Options{}) + want := RedactedSecret + ctrl + RedactedSecret + if got != want { + t.Fatalf("neighboring JWTs with %q mismatch: got %q, want %q", ctrl, got, want) + } + } + }) + + t.Run("Split key before minimum followed by slash in path", func(t *testing.T) { + // Finding 4: split key before minimum followed by slash + input := "ghp_1234567890\x0012345678901234567890123456/file" // gitleaks:allow -- synthetic redaction fixture + got := RedactString(input, Options{}) + want := RedactedSecret + "/file" + if got != want { + t.Fatalf("split before min then slash mismatch: got %q, want %q", got, want) + } + }) + + t.Run("Split key with incidental incomplete prefix", func(t *testing.T) { + // Finding 5: split key with incidental incomplete prefix glpat-1234\x00AKIA56789012 + input := "glpat-1234\x00AKIA56789012" // gitleaks:allow -- synthetic redaction fixture + got := RedactString(input, Options{}) + if got != input { + t.Fatalf("incomplete glpat with control and incomplete AKIA falsely redacted: got %q, want %q", got, input) + } + }) + + t.Run("Enclosing OpenAI key containing inner ghp suffix", func(t *testing.T) { + // Finding 6: enclosing OpenAI key containing inner ghp_ + input := "sk-abcdefghijklmnop-ghp_123456789012345678901234567890123456" // gitleaks:allow -- synthetic redaction fixture + got := RedactString(input, Options{}) + if got != RedactedSecret { + t.Fatalf("enclosing key mismatch: got %q, want %q", got, RedactedSecret) + } + }) + + t.Run("Retain left context across boundaries", func(t *testing.T) { + // Finding 7: adjacent AKIA keys + input := "AKIAIOSFODNN7EXAMPLEAKIAIOSFODNN7EXAMPLE" // gitleaks:allow -- synthetic redaction fixture + got := RedactString(input, Options{}) + want := RedactedSecret + RedactedSecret + if got != want { + t.Fatalf("adjacent AKIA keys mismatch: got %q, want %q", got, want) + } + }) + + t.Run("Preserve entire terminal control run across encodings", func(t *testing.T) { + // Finding 8: terminal control run \x00\x1b[path/file + input := "sk-ant-api03-abcdefghijklmnopqrstuvwxyz\x00\x1b[path/file" // gitleaks:allow -- synthetic redaction fixture + got := RedactString(input, Options{}) + want := RedactedSecret + "\x00\x1b[path/file" + if got != want { + t.Fatalf("terminal control run mismatch: got %q, want %q", got, want) + } + }) +} + +func TestIncompletePrefixInsideValidOpenAIKey(t *testing.T) { + first := "sk-" + strings.Repeat("a", 24) + "123456" + for _, tail := range []string{"sk-proj-abcdefg", "sk-ant-abcdefgh", "github_pat_abcd", "glpat-abcdefghi", "AIzaabcdefghijk"} { + for _, gap := range []string{"\x00", "\x1b", "\x9b", "\u009b", "\x00\x1b\u009b"} { + if got := RedactString(first+gap+tail, Options{}); got != RedactedSecret { + t.Errorf("incomplete prefix %q after gap %q leaked: %q", tail, gap, got) + } + } + } + // A complete neighbor remains independent, with the original gap intact. + second := "sk-proj-" + strings.Repeat("b", 24) + if got, want := RedactString(first+"\x00"+second, Options{}), RedactedSecret+"\x00"+RedactedSecret; got != want { + t.Errorf("complete neighboring key: got %q, want %q", got, want) + } + // An incomplete prefix still separates non-secret prose from a split key. + prose := "sk-my-awesome-kebab-project" + if got, want := RedactString(prose+"\x00sk-proj-abcdefg\x00hijklmnop12345", Options{}), prose+"\x00"+RedactedSecret; got != want { + t.Errorf("prose before split key: got %q, want %q", got, want) + } +} + +func TestNeighboringJWTLengths(t *testing.T) { + header := "eyJhbGciOiJIUzI1NiJ9" // gitleaks:allow -- synthetic redaction fixture + payloads := []string{"eyJzdWIiOiIxMjM0NTY3ODkwIn0", "eyJzdWIiOiJib2JieSIsIm5hbWUiOiJCIn0", strings.Repeat("a", 256)} // gitleaks:allow -- synthetic redaction fixture + for _, firstPayload := range payloads { + for _, secondPayload := range payloads { + a := header + "." + firstPayload + ".SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c" + b := header + "." + secondPayload + ".Qm9iYnlTaWduYXR1cmVCQkJCQkJCQkJCQg" + for _, gap := range []string{"\x00", "\x1b", "\x9b", "\u009b", "\x00\x1b\u009b"} { + if got, want := RedactString(a+gap+b, Options{}), RedactedSecret+gap+RedactedSecret; got != want { + t.Errorf("payload lengths %d/%d, gap %q: got %q, want %q", len(firstPayload), len(secondPayload), gap, got, want) + } + } + } + } +} + +func TestSplitRedactionNoCredentialSuffixRemains(t *testing.T) { + // Regression test for splits before and after minimum length: + // ensure no credential suffix is leaked in either case. + key := "sk-abcdefghijklmnopqrstuvwxyz12345678" // minOpenAILen = 23, total = 37 // gitleaks:allow -- synthetic redaction fixture + splitBeforeMin := key[:10] + "\x00" + key[10:] // pos = 10 (< 23) + splitAfterMin := key[:28] + "\x00" + key[28:] // pos = 28 (> 23) + + for _, tc := range []struct { + name string + input string + }{ + {"split before minimum length", splitBeforeMin}, + {"split after minimum length", splitAfterMin}, + } { + t.Run(tc.name, func(t *testing.T) { + got := RedactString(tc.input, Options{}) + if got != RedactedSecret { + t.Fatalf("%s leaked: got %q, want %q", tc.name, got, RedactedSecret) + } + if strings.Contains(got, key[28:]) { + t.Fatalf("%s leaked suffix %q in %q", tc.name, key[28:], got) + } + }) + } +} + +func TestSplitRedactionLargeInputs(t *testing.T) { + sizes := []int{8 * 1024, 16 * 1024, 32 * 1024, 64 * 1024, 128 * 1024} + + t.Run("OpenAI kebab repeated gaps scaling", func(t *testing.T) { + for _, size := range sizes { + var b strings.Builder + b.WriteString("sk-kebab-") // gitleaks:allow -- synthetic redaction fixture + for b.Len() < size { + b.WriteString("\x00a") + } + input := b.String() + got := RedactString(input, Options{}) + if got != input { + t.Fatalf("kebab false positive was falsely redacted at size %d", size) + } + } + }) + + t.Run("OpenAI kebab starting with bare sk- and repeated gaps scaling", func(t *testing.T) { + for _, size := range sizes { + var b strings.Builder + b.WriteString("sk-") // gitleaks:allow -- synthetic redaction fixture + for b.Len() < size { + b.WriteString("\x00a-b") + } + input := b.String() + got := RedactString(input, Options{}) + if got != input { + t.Fatalf("kebab false positive with bare sk- prefix was falsely redacted at size %d", size) + } + } + }) + + t.Run("JWT repeated gaps scaling and correct redaction", func(t *testing.T) { + for _, size := range sizes { + segLen := size / 2 + var b strings.Builder + b.WriteString("eyJ") // gitleaks:allow -- synthetic redaction fixture + for b.Len() < segLen { + b.WriteString("\x00a") + } + b.WriteString(".eyJ") + for b.Len() < size { + b.WriteString("\x00b") + } + b.WriteString(".SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c") + input := b.String() + got := RedactString(input, Options{}) + if !strings.Contains(got, RedactedSecret) { + t.Fatalf("JWT at size %d failed to redact", size) + } + } + }) + + t.Run("Anthropic repeated gaps scaling and correct redaction", func(t *testing.T) { + for _, size := range sizes { + var b strings.Builder + b.WriteString("sk-ant-api03-abcdefghijklmnopqrstuvwxyz") // gitleaks:allow -- synthetic redaction fixture + for b.Len() < size { + b.WriteString("\x00a") + } + input := b.String() + got := RedactString(input, Options{}) + if got != RedactedSecret { + t.Fatalf("Anthropic at size %d failed to redact: got %q, want %q", size, got, RedactedSecret) + } + } + }) +} + +func BenchmarkRedactJWTGaps800KB(b *testing.B) { + var builder strings.Builder + builder.WriteString("eyJ") // gitleaks:allow -- synthetic redaction fixture + for builder.Len() < 400*1024 { + builder.WriteString("\x00a") + } + builder.WriteString(".eyJ") + for builder.Len() < 800*1024 { + builder.WriteString("\x00b") + } + builder.WriteString(".SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c") + input := builder.String() + b.ResetTimer() + for i := 0; i < b.N; i++ { + _ = RedactString(input, Options{}) + } +} + +func BenchmarkRedactOpenAIKebabGaps128KB(b *testing.B) { + var builder strings.Builder + builder.WriteString("sk-kebab-") // gitleaks:allow -- synthetic redaction fixture + for builder.Len() < 128*1024 { + builder.WriteString("\x00a") + } + input := builder.String() + b.ResetTimer() + for i := 0; i < b.N; i++ { + _ = RedactString(input, Options{}) + } +} + +func BenchmarkRedactJWTGaps128KB(b *testing.B) { + var builder strings.Builder + builder.WriteString("eyJ") // gitleaks:allow -- synthetic redaction fixture + for builder.Len() < 64*1024 { + builder.WriteString("\x00a") + } + builder.WriteString(".eyJ") + for builder.Len() < 128*1024 { + builder.WriteString("\x00b") + } + builder.WriteString(".SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c") + input := builder.String() + b.ResetTimer() + for i := 0; i < b.N; i++ { + _ = RedactString(input, Options{}) + } +} + +// Keep runtime measurements out of correctness tests: race instrumentation and +// shared CI runners make absolute wall-clock deadlines unreliable. +func BenchmarkSplitRedactionScaling(b *testing.B) { + for _, size := range []int{8 << 10, 32 << 10, 128 << 10} { + inputs := map[string]string{ + "anthropic": "sk-ant-api03-" + strings.Repeat("a", 20) + strings.Repeat("\x00a", size/2), + "neighboringJWTs": strings.Repeat("eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJib2JieSIsIm5hbWUiOiJCIn0.Qm9iYnlTaWduYXR1cmVCQkJCQkJCQkJCQg\x00", size/90), // gitleaks:allow -- synthetic redaction fixture + } + for name, input := range inputs { + b.Run(name+"/"+strconv.Itoa(size), func(b *testing.B) { + b.SetBytes(int64(len(input))) + b.ReportAllocs() + for b.Loop() { + _ = RedactString(input, Options{}) + } + }) + } + } +}