From 47dccb6be552b98fa93b61b54b457b9449a12a44 Mon Sep 17 00:00:00 2001 From: alyldas Date: Mon, 13 Jul 2026 21:39:22 +0300 Subject: [PATCH] fix: expose English language pack --- README.md | 37 +- docs/external-language-pack-policy.md | 13 +- docs/language-pack-authoring.md | 4 +- examples/english-language-pack/README.md | 25 +- examples/english-language-pack/package.json | 3 - .../english-language-pack/src/dictionary.ts | 57 +- examples/english-language-pack/src/filter.ts | 493 +----------------- examples/english-language-pack/src/index.ts | 19 +- package.json | 3 +- src/index.ts | 5 + src/languages/en/dictionary.ts | 60 +++ src/languages/en/filter.ts | 492 +++++++++++++++++ src/languages/en/index.ts | 29 ++ tests/dist-public-api-smoke.mjs | 20 + tests/dist-public-api-smoke.ts | 17 + tests/english-language-pack.spec.ts | 24 +- 16 files changed, 706 insertions(+), 595 deletions(-) create mode 100644 src/languages/en/dictionary.ts create mode 100644 src/languages/en/filter.ts create mode 100644 src/languages/en/index.ts diff --git a/README.md b/README.md index e3a5fb8..3319181 100644 --- a/README.md +++ b/README.md @@ -286,8 +286,30 @@ fields. Runtime dictionary terms remain normalized literals; language dictionaries are the supported boundary for maintained language-specific rule data. Runtime calls such as `setStrict`, `setLoose`, `addStrict`, and `addLoose` only mutate the returned filter instance; they do not change the compiled -dictionary object or other filters created from it. This release intentionally -does not add another built-in language or a separately published package. +dictionary object or other filters created from it. + +The package also ships a reviewed, opt-in English language pack: + +```ts +import { + createEnglishProfanityFilter, + englishProfanityDictionary, + englishProfanityFilter, +} from "@textfilters/profanity"; + +englishProfanityFilter.check("fucking"); +englishProfanityFilter.censor("fucking"); + +const mutableEnglishFilter = createEnglishProfanityFilter(); +mutableEnglishFilter.addStrict("tenant-only-term"); +``` + +`englishProfanityFilter` is a shared read-only filter initialized on first use. +`createEnglishProfanityFilter()` returns an isolated mutable filter, while +`englishProfanityDictionary` exposes the validated source dictionary. Importing +the package does not add English rules to `filter`; the shared default remains +Russian-only. The English pack is intentionally limited to its reviewed audit +vocabulary and is not a broad toxicity or hate-speech classifier. The Russian dictionary is maintained as split family data with an explicit rule order. New high-risk family rules are expected to add nearby coverage and @@ -355,13 +377,12 @@ For future external language pack guidance, see source dictionary shape, stable ids, taxonomy metadata, strict and loose views, human-maintained JSON, and conformance expectations. The [external language pack policy](docs/external-language-pack-policy.md) -defines when the project is ready to create a real external package and keeps -the built-in Russian dictionary in this package for now. +defines when the project is ready to create a separately published language +package and keeps the built-in dictionaries in this package for now. -A reviewed, opt-in English dictionary is available as a private non-published -[English language pack example](examples/english-language-pack/README.md). It -does not change the Russian shared default and remains intentionally limited to -the reviewed audit vocabulary until English-language ownership is established. +The [English language pack example](examples/english-language-pack/README.md) +documents the reviewed vocabulary and re-exports the package-owned +implementation from `src/languages/en` for repository-level examples. ## Related Textfilters Packages diff --git a/docs/external-language-pack-policy.md b/docs/external-language-pack-policy.md index 3888c51..1496fa0 100644 --- a/docs/external-language-pack-policy.md +++ b/docs/external-language-pack-policy.md @@ -80,7 +80,12 @@ until ongoing maintenance is credible. ## Current Boundaries -The repository contains a real reviewed English dictionary as a private, -non-published example. It remains an opt-in test and authoring artifact until a -named maintainer or maintenance group satisfies the ownership criteria above. -It does not change the built-in Russian dictionary or the shared default filter. +The main package contains a reviewed English dictionary under +`src/languages/en` and exposes it through an opt-in public API. It is not a +separately published language package, does not change the built-in Russian +dictionary, and does not add English rules to the shared default filter. + +A future separately published English package still requires a named maintainer +or maintenance group that satisfies the ownership criteria above. Shipping a +limited opt-in dictionary in the main package does not by itself satisfy those +external-package requirements. diff --git a/docs/language-pack-authoring.md b/docs/language-pack-authoring.md index 959b34c..0be2e15 100644 --- a/docs/language-pack-authoring.md +++ b/docs/language-pack-authoring.md @@ -320,8 +320,8 @@ The following work is intentionally out of scope for this milestone: - creating `@textfilters/profanity-ru` or any other external language package; - moving the Russian dictionary out of this package; -- adding another real language dictionary beyond the private reviewed English - pack; +- adding another real language dictionary beyond the reviewed English pack in + `src/languages/en`; - broadening profanity coverage beyond the reviewed English vocabulary; - changing Russian runtime behavior; - adding generated matcher metadata to source dictionaries; diff --git a/examples/english-language-pack/README.md b/examples/english-language-pack/README.md index 8c9ea63..2b74d05 100644 --- a/examples/english-language-pack/README.md +++ b/examples/english-language-pack/README.md @@ -1,9 +1,9 @@ # Reviewed English Profanity Language Pack -This directory contains a real, opt-in English dictionary for the reviewed -audit vocabulary. It is deliberately non-published: the package is marked -`private`, has no release workflow, and remains an in-repository example until -an English-language maintainer or maintenance group is named. +The package-owned implementation of this reviewed, opt-in English dictionary +lives in `src/languages/en` and is included in the published distribution. This +private example re-exports that implementation so repository examples exercise +the same code as package consumers. The pack covers only `fuck`, `fucking`, `fucked`, `shit`, `dick`, `motherfucker` (including detection before a possessive suffix), `cock`, @@ -12,20 +12,26 @@ or contextual moderation. ## Opt-in API -The example exports a validated dictionary and an isolated filter: +The public package exports a validated dictionary, a shared read-only filter, +and a factory for isolated mutable filters: ```ts import { + createEnglishProfanityFilter, englishProfanityDictionary, englishProfanityFilter, -} from "./src/index.js"; +} from "@textfilters/profanity"; englishProfanityFilter.check("reviewed English text"); englishProfanityFilter.analyze("reviewed English text"); + +const mutableEnglishFilter = createEnglishProfanityFilter(); +mutableEnglishFilter.addStrict("tenant-only-term"); ``` -Importing the main `@textfilters/profanity` package does not enable these -rules. The built-in shared filter remains Russian-only. +Importing the main package does not enable these rules on the default `filter`. +The built-in shared default remains Russian-only, and the English filter is +initialized only when its methods are first called. ## Policy boundaries @@ -41,5 +47,6 @@ rules. The built-in shared filter remains Russian-only. retain the normal runtime matching contract. - The possessive suffix in `motherfucker's` is not part of the reported range; only the maintained profanity token is censored. -- Publication requires the ownership and release criteria in +- A separately published language package requires the ownership and release + criteria in [the external language pack policy](../../docs/external-language-pack-policy.md). diff --git a/examples/english-language-pack/package.json b/examples/english-language-pack/package.json index 8cb1853..bacfe57 100644 --- a/examples/english-language-pack/package.json +++ b/examples/english-language-pack/package.json @@ -6,9 +6,6 @@ "description": "Non-published reviewed English profanity language pack.", "license": "UNLICENSED", "sideEffects": false, - "dependencies": { - "@textfilters/core": "^0.3.1" - }, "files": [ "dist", "README.md" diff --git a/examples/english-language-pack/src/dictionary.ts b/examples/english-language-pack/src/dictionary.ts index fd32813..009baa5 100644 --- a/examples/english-language-pack/src/dictionary.ts +++ b/examples/english-language-pack/src/dictionary.ts @@ -1,56 +1 @@ -import type { ProfanityLanguageDictionary } from "@textfilters/profanity"; - -export const englishProfanityDictionary = { - language: "en", - rules: [ - { - id: "en.obscene.fuck.family", - category: "OBSCENE_MAT", - severity: "high", - source: "fuck(?:ed|ing)?", - match: { strict: {} }, - }, - { - id: "en.obscene.shit", - category: "OBSCENE_MAT", - severity: "high", - source: "shit", - match: { strict: {} }, - }, - { - id: "en.vulgar.dick", - category: "VULGAR", - severity: "medium", - source: "dick", - match: { strict: {} }, - }, - { - id: "en.insult.motherfucker", - category: "STRONG_INSULT", - severity: "high", - source: "motherfucker", - match: { strict: {} }, - }, - { - id: "en.vulgar.cock", - category: "VULGAR", - severity: "medium", - source: "cock", - match: { strict: {} }, - }, - { - id: "en.insult.bitch", - category: "STRONG_INSULT", - severity: "medium", - source: "bitch", - match: { strict: {} }, - }, - { - id: "en.insult.bastard", - category: "STRONG_INSULT", - severity: "medium", - source: "bastard", - match: { strict: {} }, - }, - ], -} as const satisfies ProfanityLanguageDictionary; +export { englishProfanityDictionary } from "../../../src/languages/en/dictionary.js"; diff --git a/examples/english-language-pack/src/filter.ts b/examples/english-language-pack/src/filter.ts index 0b4d88c..3d18798 100644 --- a/examples/english-language-pack/src/filter.ts +++ b/examples/english-language-pack/src/filter.ts @@ -1,492 +1 @@ -import { mergeRanges, normalizeTextInput } from "@textfilters/core"; -import type { - ProfanityFilter, - ProfanityCategory, - ProfanityMatchOptions, - ProfanityMatchRange, - ProfanitySeverity, - ProfanityTermList, -} from "../../../src/index.js"; - -import { createProfanityFilterFromDictionary } from "../../../src/index.js"; -import { normalizeLiteralTerm } from "../../../src/matchers/literals.js"; -import { normalizeForMatchSameLenWithoutHomoglyphs } from "../../../src/normalization/text.js"; -import { englishProfanityDictionary } from "./dictionary.js"; - -const DOMAIN_CHAR_RE = /[\p{L}\p{N}.-]/u; -const DOMAIN_LABEL_RE = /^[\p{L}\p{N}](?:[\p{L}\p{N}-]*[\p{L}\p{N}])?$/u; -const EMAIL_LOCAL_PART_CHAR_RE = /[\p{L}\p{N}.!#$%&'*+/=?^_`{|}~-]/u; -const EMAIL_LOCAL_PART_ATOM_RE = /^[\p{L}\p{N}!#$%&'*+/=?^_`{|}~-]+$/u; -const WORD_CHAR_RE = /[\p{L}\p{N}\p{M}_-]/u; -const URL_SCHEME_RE = /^[A-Za-z][A-Za-z0-9+.-]*$/u; -const SEVERITY_RANK: Record = { - soft: 0, - low: 1, - medium: 2, - high: 3, -}; - -interface RuntimeLiteralRecord { - readonly mode: "strict" | "loose"; - readonly category?: ProfanityCategory; - readonly severity?: ProfanitySeverity; -} - -const maintainedRuleIds = new Set( - englishProfanityDictionary.rules.map((rule) => rule.id), -); - -export const createEnglishProfanityFilter = (): ProfanityFilter => { - const baseFilter = createProfanityFilterFromDictionary( - englishProfanityDictionary, - ); - const runtimeStrict = new Map(); - const runtimeLoose = new Map(); - - const analyze = ( - text: unknown, - options?: ProfanityMatchOptions, - ): ProfanityMatchRange[] => { - const source = normalizeTextInput(text); - const normalized = normalizeForMatchSameLenWithoutHomoglyphs(source); - - return baseFilter.analyze(source).flatMap((match) => { - if (!isMaintainedRuleMatch(match)) { - return matchesTaxonomy(match, options) ? [match] : []; - } - - const candidates: ProfanityMatchRange[] = []; - if (!hasExcludedMaintainedContext(normalized, match)) { - candidates.push(match); - } - - const key = normalized.slice(match[0], match[1]).toLowerCase(); - const runtimeRecord = runtimeStrict.get(key) ?? runtimeLoose.get(key); - if (runtimeRecord !== undefined) { - candidates.push(runtimeMatchFor(match, runtimeRecord)); - } - - const selected = candidates.find((candidate) => - matchesTaxonomy(candidate, options), - ); - return selected === undefined ? [] : [selected]; - }); - }; - - return { - name: baseFilter.name, - analyze, - check: (text, options) => analyze(text, options).length > 0, - censor: (text, options) => { - const source = normalizeTextInput(text); - return maskUtf16Ranges(source, analyze(source, options)); - }, - setStrict: (list) => { - baseFilter.setStrict(list); - replaceRuntimeLiterals(runtimeStrict, list, "strict"); - }, - setLoose: (list) => { - baseFilter.setLoose(list); - replaceRuntimeLiterals(runtimeLoose, list, "loose"); - }, - addStrict: (term) => { - baseFilter.addStrict(term); - addRuntimeLiteral(runtimeStrict, term, "strict"); - }, - addLoose: (term) => { - baseFilter.addLoose(term); - addRuntimeLiteral(runtimeLoose, term, "loose"); - }, - }; -}; - -const replaceRuntimeLiterals = ( - target: Map, - terms: ProfanityTermList, - mode: RuntimeLiteralRecord["mode"], -): void => { - target.clear(); - if (!Array.isArray(terms)) { - return; - } - - for (const term of terms) { - addRuntimeLiteral(target, term, mode); - } -}; - -const addRuntimeLiteral = ( - target: Map, - term: unknown, - mode: RuntimeLiteralRecord["mode"], -): void => { - const source = runtimeLiteralSource(term); - if (source === null) { - return; - } - - const key = normalizeLiteralTerm( - source, - normalizeForMatchSameLenWithoutHomoglyphs, - ).toLowerCase(); - if (!target.has(key)) { - target.set(key, runtimeLiteralRecord(term, mode)); - } -}; - -const runtimeLiteralRecord = ( - term: unknown, - mode: RuntimeLiteralRecord["mode"], -): RuntimeLiteralRecord => { - if ( - typeof term !== "object" || - term === null || - !("source" in term) || - typeof term.source !== "string" || - term.source.trim().length === 0 - ) { - return { mode }; - } - - const category = - "category" in term - ? (term.category as ProfanityCategory | undefined) - : undefined; - const severity = - "severity" in term - ? (term.severity as ProfanitySeverity | undefined) - : undefined; - return { - mode, - ...(category === undefined ? {} : { category }), - ...(severity === undefined ? {} : { severity }), - }; -}; - -const runtimeMatchFor = ( - match: ProfanityMatchRange, - record: RuntimeLiteralRecord, -): ProfanityMatchRange => - Object.assign([match[0], match[1]] as [number, number], { - mode: record.mode, - ...(record.category === undefined ? {} : { category: record.category }), - ...(record.severity === undefined ? {} : { severity: record.severity }), - }); - -const matchesTaxonomy = ( - match: ProfanityMatchRange, - options: ProfanityMatchOptions | undefined, -): boolean => - (options?.categories === undefined || - (match.category !== undefined && - options.categories.includes(match.category))) && - (options?.severities === undefined || - (match.severity !== undefined && - options.severities.includes(match.severity))) && - (options?.minSeverity === undefined || - (match.severity !== undefined && - SEVERITY_RANK[match.severity] >= SEVERITY_RANK[options.minSeverity])); - -const runtimeLiteralSource = (term: unknown): string | null => { - const source = - typeof term === "object" && - term !== null && - "source" in term && - typeof term.source === "string" - ? term.source - : String(term ?? ""); - const trimmed = source.trim(); - return trimmed.length === 0 ? null : trimmed; -}; - -const maskUtf16Ranges = ( - source: string, - ranges: readonly ProfanityMatchRange[], -): string => { - let cursor = 0; - const parts: string[] = []; - - for (const [start, end] of mergeRanges(ranges)) { - parts.push(source.slice(cursor, start), "*".repeat(end - start)); - cursor = end; - } - - parts.push(source.slice(cursor)); - return parts.join(""); -}; - -const isMaintainedRuleMatch = (match: ProfanityMatchRange): boolean => - match.ruleId !== undefined && maintainedRuleIds.has(match.ruleId); - -const hasExcludedMaintainedContext = ( - normalized: string, - [start, end]: ProfanityMatchRange, -): boolean => - isStandaloneMention(normalized, start, end) || - isInsideMultiLabelDomain(normalized, start) || - isWithinEmailLocalPart(normalized, start, end); - -const isStandaloneMention = ( - normalized: string, - start: number, - end: number, -): boolean => { - const atIndex = start - 1; - if (atIndex < 0 || normalized[atIndex] !== "@") { - return false; - } - - if (atIndex > 0) { - const previousStart = previousCodePointStart(normalized, atIndex); - const previous = normalized.slice(previousStart, atIndex); - if (previous === "@" || WORD_CHAR_RE.test(previous)) { - return false; - } - } - - let usernameEnd = end; - while (usernameEnd < normalized.length) { - const charEnd = nextCodePointEnd(normalized, usernameEnd); - if (!WORD_CHAR_RE.test(normalized.slice(usernameEnd, charEnd))) { - break; - } - usernameEnd = charEnd; - } - - return normalized[usernameEnd] !== "@"; -}; - -const isWithinEmailLocalPart = ( - normalized: string, - matchStart: number, - matchEnd: number, -): boolean => { - let start = matchStart; - while (start > 0) { - const previousStart = previousCodePointStart(normalized, start); - if ( - !EMAIL_LOCAL_PART_CHAR_RE.test(normalized.slice(previousStart, start)) - ) { - break; - } - start = previousStart; - } - - let position = matchEnd; - - while (position < normalized.length) { - const charEnd = nextCodePointEnd(normalized, position); - if (!EMAIL_LOCAL_PART_CHAR_RE.test(normalized.slice(position, charEnd))) { - break; - } - position = charEnd; - } - - if (normalized[position] !== "@") { - return false; - } - - const localPart = normalized.slice(start, position); - return ( - hasEmailLocalPartBoundary(normalized, start) && - isValidDotAtomLocalPart(localPart) && - isMultiLabelDomainAt(normalized, position + 1) - ); -}; - -const hasEmailLocalPartBoundary = ( - normalized: string, - start: number, -): boolean => - start === 0 || - normalized.slice(previousCodePointStart(normalized, start), start) !== "@"; - -const isValidDotAtomLocalPart = (candidate: string): boolean => - candidate.length <= 64 && - candidate - .split(".") - .every((atom) => atom.length > 0 && EMAIL_LOCAL_PART_ATOM_RE.test(atom)); - -const isInsideMultiLabelDomain = ( - normalized: string, - position: number, -): boolean => { - let start = position; - while (start > 0) { - const previousStart = previousCodePointStart(normalized, start); - if (!DOMAIN_CHAR_RE.test(normalized.slice(previousStart, start))) { - break; - } - start = previousStart; - } - - return isMultiLabelDomainAt(normalized, start, true); -}; - -const isMultiLabelDomainAt = ( - normalized: string, - start: number, - trimLeadingPunctuation = false, -): boolean => { - let end = start; - while (end < normalized.length) { - const charEnd = nextCodePointEnd(normalized, end); - if (!DOMAIN_CHAR_RE.test(normalized.slice(end, charEnd))) { - break; - } - end = charEnd; - } - - if ( - start === end || - normalized[end] === "@" || - !hasValidDomainStartContext(normalized, start) || - hasWordContinuation(normalized, start, end) || - !isValidMultiLabelDomain( - normalized.slice(start, end), - trimLeadingPunctuation, - ) - ) { - return false; - } - - return true; -}; - -const hasValidDomainStartContext = ( - normalized: string, - start: number, -): boolean => { - if (start === 0) { - return true; - } - - const previousStart = previousCodePointStart(normalized, start); - const previous = normalized.slice(previousStart, start); - if (previous === "/") { - return normalized[start] !== "." && isUrlAuthorityStart(normalized, start); - } - if (previous === "\\") { - return false; - } - if (previous !== "@") { - return !hasPathSeparatorInCurrentToken(normalized, start); - } - - if (normalized[start] === ".") { - return false; - } - - let localStart = previousStart; - while (localStart > 0) { - const charStart = previousCodePointStart(normalized, localStart); - if ( - !EMAIL_LOCAL_PART_CHAR_RE.test(normalized.slice(charStart, localStart)) - ) { - break; - } - localStart = charStart; - } - - return ( - hasEmailLocalPartBoundary(normalized, localStart) && - isValidDotAtomLocalPart(normalized.slice(localStart, previousStart)) - ); -}; - -const hasPathSeparatorInCurrentToken = ( - normalized: string, - start: number, -): boolean => { - let position = start; - while (position > 0) { - const charStart = previousCodePointStart(normalized, position); - const char = normalized.slice(charStart, position); - if (/\s/u.test(char)) { - return false; - } - if (char === "?" || char === "#" || char === "=") { - return false; - } - if (char === "/" || char === "\\") { - return true; - } - position = charStart; - } - return false; -}; - -const isUrlAuthorityStart = (normalized: string, start: number): boolean => { - const firstSlash = start - 2; - if (firstSlash < 0 || normalized[firstSlash] !== "/") { - return false; - } - if (firstSlash === 0) { - return true; - } - - const prefixStart = previousCodePointStart(normalized, firstSlash); - const prefix = normalized.slice(prefixStart, firstSlash); - if (prefix === ":") { - let schemeStart = prefixStart; - while (schemeStart > 0) { - const charStart = previousCodePointStart(normalized, schemeStart); - if (!/[A-Za-z0-9+.-]/u.test(normalized.slice(charStart, schemeStart))) { - break; - } - schemeStart = charStart; - } - return URL_SCHEME_RE.test(normalized.slice(schemeStart, prefixStart)); - } - - return prefix !== "/" && prefix !== "\\" && !WORD_CHAR_RE.test(prefix); -}; - -const hasWordContinuation = ( - normalized: string, - start: number, - end: number, -): boolean => { - const before = - start === 0 - ? "" - : normalized.slice(previousCodePointStart(normalized, start), start); - const after = - end === normalized.length - ? "" - : normalized.slice(end, nextCodePointEnd(normalized, end)); - - return WORD_CHAR_RE.test(before) || WORD_CHAR_RE.test(after); -}; - -const isValidMultiLabelDomain = ( - candidate: string, - trimLeadingPunctuation: boolean, -): boolean => { - const withoutTrailingPunctuation = candidate.replace(/\.+$/u, ""); - const domain = trimLeadingPunctuation - ? withoutTrailingPunctuation.replace(/^\.+/u, "") - : withoutTrailingPunctuation; - const labels = domain.split("."); - return ( - domain.length <= 253 && - labels.length >= 2 && - labels.every((label) => label.length <= 63 && DOMAIN_LABEL_RE.test(label)) - ); -}; - -const nextCodePointEnd = (value: string, index: number): number => { - const code = value.codePointAt(index); - return index + (code !== undefined && code > 0xffff ? 2 : 1); -}; - -const previousCodePointStart = (value: string, index: number): number => { - const previous = Math.max(0, index - 1); - return previous > 0 && - value.charCodeAt(previous) >= 0xdc00 && - value.charCodeAt(previous) <= 0xdfff && - value.charCodeAt(previous - 1) >= 0xd800 && - value.charCodeAt(previous - 1) <= 0xdbff - ? previous - 1 - : previous; -}; +export { createEnglishProfanityFilter } from "../../../src/languages/en/filter.js"; diff --git a/examples/english-language-pack/src/index.ts b/examples/english-language-pack/src/index.ts index 268b92f..e7b1bed 100644 --- a/examples/english-language-pack/src/index.ts +++ b/examples/english-language-pack/src/index.ts @@ -1,16 +1,5 @@ -import { validateProfanityLanguageDictionary } from "../../../src/index.js"; - -import { englishProfanityDictionary } from "./dictionary.js"; -import { createEnglishProfanityFilter } from "./filter.js"; - -const validationIssues = validateProfanityLanguageDictionary( +export { + createEnglishProfanityFilter, englishProfanityDictionary, -); - -if (validationIssues.length > 0) { - throw new Error(JSON.stringify(validationIssues, null, 2)); -} - -export { englishProfanityDictionary }; -export { createEnglishProfanityFilter }; -export const englishProfanityFilter = createEnglishProfanityFilter(); + englishProfanityFilter, +} from "../../../src/index.js"; diff --git a/package.json b/package.json index 511aa7b..eaf0554 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "name": "@textfilters/profanity", "version": "0.15.0", "type": "module", - "description": "TypeScript profanity filter with Russian dictionary support, obfuscation handling, taxonomy metadata, and language-pack validation.", + "description": "TypeScript profanity filter with Russian and opt-in English dictionaries, obfuscation handling, taxonomy metadata, and language-pack validation.", "license": "MIT", "keywords": [ "text-filtering", @@ -14,6 +14,7 @@ "redaction", "profanity-filter", "russian-profanity", + "english-profanity", "language-dictionaries", "obfuscation", "taxonomy", diff --git a/src/index.ts b/src/index.ts index 8e55d73..f36bd26 100644 --- a/src/index.ts +++ b/src/index.ts @@ -41,4 +41,9 @@ export { } from "./filter.js"; export type { CompiledProfanityDictionary } from "./filter.js"; export { createProfanityScanner } from "./scanner.js"; +export { + createEnglishProfanityFilter, + englishProfanityDictionary, + englishProfanityFilter, +} from "./languages/en/index.js"; export { russianProfanityDictionary } from "./languages/ru/index.js"; diff --git a/src/languages/en/dictionary.ts b/src/languages/en/dictionary.ts new file mode 100644 index 0000000..00e9334 --- /dev/null +++ b/src/languages/en/dictionary.ts @@ -0,0 +1,60 @@ +import type { ProfanityLanguageDictionary } from "../profanity.js"; + +export const createReviewedEnglishProfanityDictionary = () => + ({ + language: "en", + rules: [ + { + id: "en.obscene.fuck.family", + category: "OBSCENE_MAT", + severity: "high", + source: "fuck(?:ed|ing)?", + match: { strict: {} }, + }, + { + id: "en.obscene.shit", + category: "OBSCENE_MAT", + severity: "high", + source: "shit", + match: { strict: {} }, + }, + { + id: "en.vulgar.dick", + category: "VULGAR", + severity: "medium", + source: "dick", + match: { strict: {} }, + }, + { + id: "en.insult.motherfucker", + category: "STRONG_INSULT", + severity: "high", + source: "motherfucker", + match: { strict: {} }, + }, + { + id: "en.vulgar.cock", + category: "VULGAR", + severity: "medium", + source: "cock", + match: { strict: {} }, + }, + { + id: "en.insult.bitch", + category: "STRONG_INSULT", + severity: "medium", + source: "bitch", + match: { strict: {} }, + }, + { + id: "en.insult.bastard", + category: "STRONG_INSULT", + severity: "medium", + source: "bastard", + match: { strict: {} }, + }, + ], + }) as const satisfies ProfanityLanguageDictionary; + +export const englishProfanityDictionary = + createReviewedEnglishProfanityDictionary(); diff --git a/src/languages/en/filter.ts b/src/languages/en/filter.ts new file mode 100644 index 0000000..3ebc81e --- /dev/null +++ b/src/languages/en/filter.ts @@ -0,0 +1,492 @@ +import { mergeRanges, normalizeTextInput } from "@textfilters/core"; +import type { + ProfanityCategory, + ProfanityFilter, + ProfanityMatchOptions, + ProfanityMatchRange, + ProfanitySeverity, + ProfanityTermList, +} from "../../types.js"; + +import { createProfanityFilterFromDictionary } from "../../filter.js"; +import { normalizeLiteralTerm } from "../../matchers/literals.js"; +import { normalizeForMatchSameLenWithoutHomoglyphs } from "../../normalization/text.js"; +import { createReviewedEnglishProfanityDictionary } from "./dictionary.js"; + +const DOMAIN_CHAR_RE = /[\p{L}\p{N}.-]/u; +const DOMAIN_LABEL_RE = /^[\p{L}\p{N}](?:[\p{L}\p{N}-]*[\p{L}\p{N}])?$/u; +const EMAIL_LOCAL_PART_CHAR_RE = /[\p{L}\p{N}.!#$%&'*+/=?^_`{|}~-]/u; +const EMAIL_LOCAL_PART_ATOM_RE = /^[\p{L}\p{N}!#$%&'*+/=?^_`{|}~-]+$/u; +const WORD_CHAR_RE = /[\p{L}\p{N}\p{M}_-]/u; +const URL_SCHEME_RE = /^[A-Za-z][A-Za-z0-9+.-]*$/u; +const SEVERITY_RANK: Record = { + soft: 0, + low: 1, + medium: 2, + high: 3, +}; + +interface RuntimeLiteralRecord { + readonly mode: "strict" | "loose"; + readonly category?: ProfanityCategory; + readonly severity?: ProfanitySeverity; +} + +const maintainedRuleIds = new Set( + createReviewedEnglishProfanityDictionary().rules.map((rule) => rule.id), +); + +export const createEnglishProfanityFilter = (): ProfanityFilter => { + const baseFilter = createProfanityFilterFromDictionary( + createReviewedEnglishProfanityDictionary(), + ); + const runtimeStrict = new Map(); + const runtimeLoose = new Map(); + + const analyze = ( + text: unknown, + options?: ProfanityMatchOptions, + ): ProfanityMatchRange[] => { + const source = normalizeTextInput(text); + const normalized = normalizeForMatchSameLenWithoutHomoglyphs(source); + + return baseFilter.analyze(source).flatMap((match) => { + if (!isMaintainedRuleMatch(match)) { + return matchesTaxonomy(match, options) ? [match] : []; + } + + const candidates: ProfanityMatchRange[] = []; + if (!hasExcludedMaintainedContext(normalized, match)) { + candidates.push(match); + } + + const key = normalized.slice(match[0], match[1]).toLowerCase(); + const runtimeRecord = runtimeStrict.get(key) ?? runtimeLoose.get(key); + if (runtimeRecord !== undefined) { + candidates.push(runtimeMatchFor(match, runtimeRecord)); + } + + const selected = candidates.find((candidate) => + matchesTaxonomy(candidate, options), + ); + return selected === undefined ? [] : [selected]; + }); + }; + + return { + name: baseFilter.name, + analyze, + check: (text, options) => analyze(text, options).length > 0, + censor: (text, options) => { + const source = normalizeTextInput(text); + return maskUtf16Ranges(source, analyze(source, options)); + }, + setStrict: (list) => { + baseFilter.setStrict(list); + replaceRuntimeLiterals(runtimeStrict, list, "strict"); + }, + setLoose: (list) => { + baseFilter.setLoose(list); + replaceRuntimeLiterals(runtimeLoose, list, "loose"); + }, + addStrict: (term) => { + baseFilter.addStrict(term); + addRuntimeLiteral(runtimeStrict, term, "strict"); + }, + addLoose: (term) => { + baseFilter.addLoose(term); + addRuntimeLiteral(runtimeLoose, term, "loose"); + }, + }; +}; + +const replaceRuntimeLiterals = ( + target: Map, + terms: ProfanityTermList, + mode: RuntimeLiteralRecord["mode"], +): void => { + target.clear(); + if (!Array.isArray(terms)) { + return; + } + + for (const term of terms) { + addRuntimeLiteral(target, term, mode); + } +}; + +const addRuntimeLiteral = ( + target: Map, + term: unknown, + mode: RuntimeLiteralRecord["mode"], +): void => { + const source = runtimeLiteralSource(term); + if (source === null) { + return; + } + + const key = normalizeLiteralTerm( + source, + normalizeForMatchSameLenWithoutHomoglyphs, + ).toLowerCase(); + if (!target.has(key)) { + target.set(key, runtimeLiteralRecord(term, mode)); + } +}; + +const runtimeLiteralRecord = ( + term: unknown, + mode: RuntimeLiteralRecord["mode"], +): RuntimeLiteralRecord => { + if ( + typeof term !== "object" || + term === null || + !("source" in term) || + typeof term.source !== "string" || + term.source.trim().length === 0 + ) { + return { mode }; + } + + const category = + "category" in term + ? (term.category as ProfanityCategory | undefined) + : undefined; + const severity = + "severity" in term + ? (term.severity as ProfanitySeverity | undefined) + : undefined; + return { + mode, + ...(category === undefined ? {} : { category }), + ...(severity === undefined ? {} : { severity }), + }; +}; + +const runtimeMatchFor = ( + match: ProfanityMatchRange, + record: RuntimeLiteralRecord, +): ProfanityMatchRange => + Object.assign([match[0], match[1]] as [number, number], { + mode: record.mode, + ...(record.category === undefined ? {} : { category: record.category }), + ...(record.severity === undefined ? {} : { severity: record.severity }), + }); + +const matchesTaxonomy = ( + match: ProfanityMatchRange, + options: ProfanityMatchOptions | undefined, +): boolean => + (options?.categories === undefined || + (match.category !== undefined && + options.categories.includes(match.category))) && + (options?.severities === undefined || + (match.severity !== undefined && + options.severities.includes(match.severity))) && + (options?.minSeverity === undefined || + (match.severity !== undefined && + SEVERITY_RANK[match.severity] >= SEVERITY_RANK[options.minSeverity])); + +const runtimeLiteralSource = (term: unknown): string | null => { + const source = + typeof term === "object" && + term !== null && + "source" in term && + typeof term.source === "string" + ? term.source + : String(term ?? ""); + const trimmed = source.trim(); + return trimmed.length === 0 ? null : trimmed; +}; + +const maskUtf16Ranges = ( + source: string, + ranges: readonly ProfanityMatchRange[], +): string => { + let cursor = 0; + const parts: string[] = []; + + for (const [start, end] of mergeRanges(ranges)) { + parts.push(source.slice(cursor, start), "*".repeat(end - start)); + cursor = end; + } + + parts.push(source.slice(cursor)); + return parts.join(""); +}; + +const isMaintainedRuleMatch = (match: ProfanityMatchRange): boolean => + match.ruleId !== undefined && maintainedRuleIds.has(match.ruleId); + +const hasExcludedMaintainedContext = ( + normalized: string, + [start, end]: ProfanityMatchRange, +): boolean => + isStandaloneMention(normalized, start, end) || + isInsideMultiLabelDomain(normalized, start) || + isWithinEmailLocalPart(normalized, start, end); + +const isStandaloneMention = ( + normalized: string, + start: number, + end: number, +): boolean => { + const atIndex = start - 1; + if (atIndex < 0 || normalized[atIndex] !== "@") { + return false; + } + + if (atIndex > 0) { + const previousStart = previousCodePointStart(normalized, atIndex); + const previous = normalized.slice(previousStart, atIndex); + if (previous === "@" || WORD_CHAR_RE.test(previous)) { + return false; + } + } + + let usernameEnd = end; + while (usernameEnd < normalized.length) { + const charEnd = nextCodePointEnd(normalized, usernameEnd); + if (!WORD_CHAR_RE.test(normalized.slice(usernameEnd, charEnd))) { + break; + } + usernameEnd = charEnd; + } + + return normalized[usernameEnd] !== "@"; +}; + +const isWithinEmailLocalPart = ( + normalized: string, + matchStart: number, + matchEnd: number, +): boolean => { + let start = matchStart; + while (start > 0) { + const previousStart = previousCodePointStart(normalized, start); + if ( + !EMAIL_LOCAL_PART_CHAR_RE.test(normalized.slice(previousStart, start)) + ) { + break; + } + start = previousStart; + } + + let position = matchEnd; + + while (position < normalized.length) { + const charEnd = nextCodePointEnd(normalized, position); + if (!EMAIL_LOCAL_PART_CHAR_RE.test(normalized.slice(position, charEnd))) { + break; + } + position = charEnd; + } + + if (normalized[position] !== "@") { + return false; + } + + const localPart = normalized.slice(start, position); + return ( + hasEmailLocalPartBoundary(normalized, start) && + isValidDotAtomLocalPart(localPart) && + isMultiLabelDomainAt(normalized, position + 1) + ); +}; + +const hasEmailLocalPartBoundary = ( + normalized: string, + start: number, +): boolean => + start === 0 || + normalized.slice(previousCodePointStart(normalized, start), start) !== "@"; + +const isValidDotAtomLocalPart = (candidate: string): boolean => + candidate.length <= 64 && + candidate + .split(".") + .every((atom) => atom.length > 0 && EMAIL_LOCAL_PART_ATOM_RE.test(atom)); + +const isInsideMultiLabelDomain = ( + normalized: string, + position: number, +): boolean => { + let start = position; + while (start > 0) { + const previousStart = previousCodePointStart(normalized, start); + if (!DOMAIN_CHAR_RE.test(normalized.slice(previousStart, start))) { + break; + } + start = previousStart; + } + + return isMultiLabelDomainAt(normalized, start, true); +}; + +const isMultiLabelDomainAt = ( + normalized: string, + start: number, + trimLeadingPunctuation = false, +): boolean => { + let end = start; + while (end < normalized.length) { + const charEnd = nextCodePointEnd(normalized, end); + if (!DOMAIN_CHAR_RE.test(normalized.slice(end, charEnd))) { + break; + } + end = charEnd; + } + + if ( + start === end || + normalized[end] === "@" || + !hasValidDomainStartContext(normalized, start) || + hasWordContinuation(normalized, start, end) || + !isValidMultiLabelDomain( + normalized.slice(start, end), + trimLeadingPunctuation, + ) + ) { + return false; + } + + return true; +}; + +const hasValidDomainStartContext = ( + normalized: string, + start: number, +): boolean => { + if (start === 0) { + return true; + } + + const previousStart = previousCodePointStart(normalized, start); + const previous = normalized.slice(previousStart, start); + if (previous === "/") { + return normalized[start] !== "." && isUrlAuthorityStart(normalized, start); + } + if (previous === "\\") { + return false; + } + if (previous !== "@") { + return !hasPathSeparatorInCurrentToken(normalized, start); + } + + if (normalized[start] === ".") { + return false; + } + + let localStart = previousStart; + while (localStart > 0) { + const charStart = previousCodePointStart(normalized, localStart); + if ( + !EMAIL_LOCAL_PART_CHAR_RE.test(normalized.slice(charStart, localStart)) + ) { + break; + } + localStart = charStart; + } + + return ( + hasEmailLocalPartBoundary(normalized, localStart) && + isValidDotAtomLocalPart(normalized.slice(localStart, previousStart)) + ); +}; + +const hasPathSeparatorInCurrentToken = ( + normalized: string, + start: number, +): boolean => { + let position = start; + while (position > 0) { + const charStart = previousCodePointStart(normalized, position); + const char = normalized.slice(charStart, position); + if (/\s/u.test(char)) { + return false; + } + if (char === "?" || char === "#" || char === "=") { + return false; + } + if (char === "/" || char === "\\") { + return true; + } + position = charStart; + } + return false; +}; + +const isUrlAuthorityStart = (normalized: string, start: number): boolean => { + const firstSlash = start - 2; + if (firstSlash < 0 || normalized[firstSlash] !== "/") { + return false; + } + if (firstSlash === 0) { + return true; + } + + const prefixStart = previousCodePointStart(normalized, firstSlash); + const prefix = normalized.slice(prefixStart, firstSlash); + if (prefix === ":") { + let schemeStart = prefixStart; + while (schemeStart > 0) { + const charStart = previousCodePointStart(normalized, schemeStart); + if (!/[A-Za-z0-9+.-]/u.test(normalized.slice(charStart, schemeStart))) { + break; + } + schemeStart = charStart; + } + return URL_SCHEME_RE.test(normalized.slice(schemeStart, prefixStart)); + } + + return prefix !== "/" && prefix !== "\\" && !WORD_CHAR_RE.test(prefix); +}; + +const hasWordContinuation = ( + normalized: string, + start: number, + end: number, +): boolean => { + const before = + start === 0 + ? "" + : normalized.slice(previousCodePointStart(normalized, start), start); + const after = + end === normalized.length + ? "" + : normalized.slice(end, nextCodePointEnd(normalized, end)); + + return WORD_CHAR_RE.test(before) || WORD_CHAR_RE.test(after); +}; + +const isValidMultiLabelDomain = ( + candidate: string, + trimLeadingPunctuation: boolean, +): boolean => { + const withoutTrailingPunctuation = candidate.replace(/\.+$/u, ""); + const domain = trimLeadingPunctuation + ? withoutTrailingPunctuation.replace(/^\.+/u, "") + : withoutTrailingPunctuation; + const labels = domain.split("."); + return ( + domain.length <= 253 && + labels.length >= 2 && + labels.every((label) => label.length <= 63 && DOMAIN_LABEL_RE.test(label)) + ); +}; + +const nextCodePointEnd = (value: string, index: number): number => { + const code = value.codePointAt(index); + return index + (code !== undefined && code > 0xffff ? 2 : 1); +}; + +const previousCodePointStart = (value: string, index: number): number => { + const previous = Math.max(0, index - 1); + return previous > 0 && + value.charCodeAt(previous) >= 0xdc00 && + value.charCodeAt(previous) <= 0xdfff && + value.charCodeAt(previous - 1) >= 0xd800 && + value.charCodeAt(previous - 1) <= 0xdbff + ? previous - 1 + : previous; +}; diff --git a/src/languages/en/index.ts b/src/languages/en/index.ts new file mode 100644 index 0000000..ac270ce --- /dev/null +++ b/src/languages/en/index.ts @@ -0,0 +1,29 @@ +import { + PROFANITY_FILTER_NAME, + type ProfanityFilter, + type ProfanityMatchOptions, + type ReadonlyProfanityFilter, +} from "../../types.js"; + +import { englishProfanityDictionary } from "./dictionary.js"; +import { createEnglishProfanityFilter } from "./filter.js"; + +let sharedEnglishFilter: ProfanityFilter | undefined; + +function getSharedEnglishFilter(): ProfanityFilter { + sharedEnglishFilter ??= createEnglishProfanityFilter(); + return sharedEnglishFilter; +} + +export { englishProfanityDictionary }; +export { createEnglishProfanityFilter }; + +export const englishProfanityFilter: ReadonlyProfanityFilter = Object.freeze({ + name: PROFANITY_FILTER_NAME, + analyze: (text: unknown, options?: ProfanityMatchOptions) => + getSharedEnglishFilter().analyze(text, options), + check: (text: unknown, options?: ProfanityMatchOptions) => + getSharedEnglishFilter().check(text, options), + censor: (text: unknown, options?: ProfanityMatchOptions) => + getSharedEnglishFilter().censor(text, options), +}); diff --git a/tests/dist-public-api-smoke.mjs b/tests/dist-public-api-smoke.mjs index efb9627..1d9661b 100644 --- a/tests/dist-public-api-smoke.mjs +++ b/tests/dist-public-api-smoke.mjs @@ -1,8 +1,11 @@ import { compileProfanityDictionary, + createEnglishProfanityFilter, createProfanityFilter, createProfanityFilterFromCompiledDictionary, createProfanityFilterFromDictionary, + englishProfanityDictionary, + englishProfanityFilter, filter, PROFANITY_FILTER_NAME, russianProfanityDictionary, @@ -35,6 +38,23 @@ if (!filter.check("привет блядь")) { throw new Error("Default dist filter did not detect a built-in match."); } +if ( + englishProfanityDictionary.language !== "en" || + !englishProfanityFilter.check("fucking") || + "addStrict" in englishProfanityFilter +) { + throw new Error("Dist English language pack was not exported read-only."); +} + +const mutableEnglishFilter = createEnglishProfanityFilter(); +mutableEnglishFilter.addStrict("tenant-only-term"); + +if (!mutableEnglishFilter.check("tenant-only-term")) { + throw new Error( + "Dist English filter factory did not return a mutable filter.", + ); +} + if (!dictionaryFilter.check("привет блядь")) { throw new Error("Dist dictionary filter did not detect a built-in match."); } diff --git a/tests/dist-public-api-smoke.ts b/tests/dist-public-api-smoke.ts index f36bed6..70d04d0 100644 --- a/tests/dist-public-api-smoke.ts +++ b/tests/dist-public-api-smoke.ts @@ -1,10 +1,13 @@ import type { TextCodePointRange } from "@textfilters/core"; import { compileProfanityDictionary, + createEnglishProfanityFilter, createProfanityFilter, createProfanityFilterFromCompiledDictionary, createProfanityFilterFromDictionary, createProfanityScanner, + englishProfanityDictionary, + englishProfanityFilter, filter, PROFANITY_FILTER_NAME, russianProfanityDictionary, @@ -51,6 +54,8 @@ const metadata: ProfanityTaxonomyMetadata = { severity, }; const dictionary: ProfanityLanguageDictionary = russianProfanityDictionary; +const englishDictionary: ProfanityLanguageDictionary = + englishProfanityDictionary; const compiledDictionary: CompiledProfanityDictionary = compileProfanityDictionary(dictionary); const dictionaryRule: ProfanityLanguageRuleDefinition | undefined = @@ -71,6 +76,8 @@ const dictionaryFilter: ProfanityFilter = const compiledDictionaryFilter: ProfanityFilter = createProfanityFilterFromCompiledDictionary(compiledDictionary); const sharedFilter: ReadonlyProfanityFilter = filter; +const sharedEnglishFilter: ReadonlyProfanityFilter = englishProfanityFilter; +const mutableEnglishFilter: ProfanityFilter = createEnglishProfanityFilter(); const scannerOptions: ProfanityScannerOptions = { filter: strict, matchOptions: options, @@ -100,6 +107,8 @@ const scanMetadata: ProfanityScannerMetadata = scanResult.metadata; filter.check("plain text"); sharedFilter.censor("plain text"); +sharedEnglishFilter.censor("fucking text"); +mutableEnglishFilter.addStrict("tenant-only-term"); dictionaryFilter.check("plain text"); compiledDictionaryFilter.check("plain text"); @@ -118,6 +127,14 @@ if (dictionaryValidationIssues.length !== 0) { throw new Error("Unexpected dictionary validation issue surface."); } +if ( + englishDictionary.language !== "en" || + !sharedEnglishFilter.check("fucking") || + "addStrict" in sharedEnglishFilter +) { + throw new Error("Unexpected English language pack declaration surface."); +} + if ( compiledDictionary.language !== "ru" || compiledDictionary.strictRuleCount === 0 || diff --git a/tests/english-language-pack.spec.ts b/tests/english-language-pack.spec.ts index 5035322..a721057 100644 --- a/tests/english-language-pack.spec.ts +++ b/tests/english-language-pack.spec.ts @@ -1,16 +1,14 @@ import { describe, expect, it } from "vitest"; import { + createEnglishProfanityFilter, createProfanityFilterFromDictionary, createProfanityScanner, + englishProfanityDictionary, + englishProfanityFilter, filter, validateProfanityLanguageDictionary, } from "../src"; -import { - createEnglishProfanityFilter, - englishProfanityDictionary, - englishProfanityFilter, -} from "../examples/english-language-pack/src/index.js"; const reviewedCases = [ ["fuck", "en.obscene.fuck.family", "OBSCENE_MAT", "high"], @@ -212,6 +210,22 @@ describe("reviewed English language pack", () => { expect(filter.check("fucking")).toBe(false); }); + it("exports a read-only shared English filter", () => { + const mutableExportedRules = + englishProfanityDictionary.rules as unknown as unknown[]; + const originalRules = [...mutableExportedRules]; + mutableExportedRules.length = 0; + + try { + expect(englishProfanityFilter.check("fucking")).toBe(true); + expect(createEnglishProfanityFilter().check("fucking")).toBe(true); + expect("addStrict" in englishProfanityFilter).toBe(false); + expect(Object.isFrozen(englishProfanityFilter)).toBe(true); + } finally { + mutableExportedRules.push(...originalRules); + } + }); + it("uses English normalization for runtime literal mutations", () => { const mutable = createEnglishProfanityFilter();