Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,15 @@
lockkit has no stability promise before v1: pin an exact version.
GUIDE.md is the contract between lockkit and the products built on it.

## Unreleased

- pii: `LooksPersonNameColumn` reads German plurals in `-er`
(`mitglieder`) and `staedte`/`städte` as table hints. The detection
bench scores it on sampled name columns (`namecolumns.jsonl`:
`name`/`full_name`/`nom`/`naam` with EN/FR/NL/DE names, and
`products`/`roles`/`cities`/`tags` negatives), held to the baseline
like the detectors (#163).

## v0.19.1 (2026-11-13)

### Changes
Expand Down
82 changes: 79 additions & 3 deletions pii/bench_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ const (
benchMinPersonal = 20
benchMinQuasi = 10
benchMinNeither = 15
benchMinColumns = 30 // name columns per language, each label
benchOutsideShare = 0.30
)

Expand Down Expand Up @@ -164,7 +165,7 @@ func (r benchReport) metrics() map[string]float64 {
return out
}

type confusion struct{ TP, FP, FN int }
type confusion struct{ TP, FP, FN, TN int }

func (c confusion) precision() float64 {
if c.TP+c.FP == 0 {
Expand Down Expand Up @@ -192,6 +193,8 @@ func scoreFields(cases []gen.FieldCase) (map[string]map[string]*confusion, map[s
c.FP++
case want:
c.FN++
default:
c.TN++
}
}
}
Expand All @@ -209,6 +212,36 @@ func scoreFields(cases []gen.FieldCase) (map[string]map[string]*confusion, map[s
return rep, out
}

// scoreColumns scores LooksPersonNameColumn on the name column cases, per
// language and overall: namecolumns/lang/precision and /recall.
func scoreColumns(cases []gen.ColumnCase) (map[string]*confusion, map[string]float64) {
rep := map[string]*confusion{"all": {}}
for _, c := range cases {
got := LooksPersonNameColumn(c.Table, c.Column, c.Samples)
for _, k := range []string{c.Lang, "all"} {
if rep[k] == nil {
rep[k] = &confusion{}
}
switch {
case got && c.PersonNames:
rep[k].TP++
case got:
rep[k].FP++
case c.PersonNames:
rep[k].FN++
default:
rep[k].TN++
}
}
}
out := map[string]float64{}
for lang, c := range rep {
out["namecolumns/"+lang+"/precision"] = round3(c.precision())
out["namecolumns/"+lang+"/recall"] = round3(c.recall())
}
return rep, out
}

// compareBaseline returns the metrics below their baseline, missing or
// not in it (failures), and those above it (rises).
func compareBaseline(cur, base map[string]float64) (fails, rises []string) {
Expand Down Expand Up @@ -284,18 +317,23 @@ func benchDetectors(t *testing.T) []Detector {
type benchRun struct {
texts benchReport
fields map[string]map[string]*confusion
columns map[string]*confusion
metrics map[string]float64
}

func runBench(t *testing.T) benchRun {
t.Helper()
texts := scoreBench(benchTexts(t), benchDetectors(t))
fields, fm := scoreFields(readJSONL[gen.FieldCase](t, filepath.Join(benchDir, "fields.jsonl")))
columns, cm := scoreColumns(readJSONL[gen.ColumnCase](t, filepath.Join(benchDir, "namecolumns.jsonl")))
m := texts.metrics()
for k, v := range fm {
m[k] = v
}
return benchRun{texts, fields, m}
for k, v := range cm {
m[k] = v
}
return benchRun{texts, fields, columns, m}
}

func readBaseline(t *testing.T) map[string]float64 {
Expand Down Expand Up @@ -384,6 +422,13 @@ func (run benchRun) results() string {
fmt.Fprintf(&b, "| `%s` | %s | %.3f | %.3f |\n", fn, lang, c.precision(), c.recall())
}
}
b.WriteString("\n## Name columns\n\n`LooksPersonNameColumn` on sampled columns whose name means \"name\": ")
b.WriteString("person names in tables of people or neutral tables, things, places, labels and companies otherwise.\n\n")
b.WriteString("| Lang | Columns | Precision | Recall |\n|---|--:|--:|--:|\n")
for _, lang := range append(slices.Clone(gen.Langs), "all") {
c := run.columns[lang]
fmt.Fprintf(&b, "| %s | %d | %.3f | %.3f |\n", lang, c.TP+c.FP+c.FN+c.TN, c.precision(), c.recall())
}
b.WriteString("\n## Known gaps\n\nEvery detector and language with a recall below 0.9:\n\n")
if len(gaps) == 0 {
b.WriteString("None.\n")
Expand Down Expand Up @@ -534,6 +579,21 @@ func TestBenchCorpusCoverage(t *testing.T) {
}
fields[f.Lang] = n
}
columns := map[string][2]int{}
for _, c := range readJSONL[gen.ColumnCase](t, filepath.Join(benchDir, "namecolumns.jsonl")) {
n := columns[c.Lang]
if c.PersonNames {
n[0]++
} else {
n[1]++
}
columns[c.Lang] = n
}
for _, lang := range gen.Langs {
if n := columns[lang]; n[0] < benchMinColumns || n[1] < benchMinColumns {
t.Errorf("%s name columns: %d of person names, %d not; want at least %d each", lang, n[0], n[1], benchMinColumns)
}
}
for _, lang := range gen.Langs {
if n := fields[lang]; n[0] < benchMinPersonal || n[1] < benchMinQuasi || n[2] < benchMinNeither {
t.Errorf("%s fields: %d personal, %d quasi, %d neither; want at least %d, %d, %d",
Expand Down Expand Up @@ -608,14 +668,30 @@ func TestScoreFields(t *testing.T) {
{Lang: "en", Field: "birth_date", Quasi: true},
{Lang: "fr", Field: "file_name", Personal: true}, // labelled personal, not found
})
if got, want := *rep["personal"]["all"], (confusion{TP: 1, FN: 1}); got != want {
if got, want := *rep["personal"]["all"], (confusion{TP: 1, FN: 1, TN: 2}); got != want {
t.Errorf("personal: %+v, want %+v", got, want)
}
if m["fields/personal/all/recall"] != 0.5 || m["fields/personal/all/precision"] != 1 || m["fields/quasi/en/recall"] != 1 {
t.Errorf("metrics %v", m)
}
}

func TestScoreColumns(t *testing.T) {
rep, m := scoreColumns([]gen.ColumnCase{
{Lang: "fr", Table: "clients", Column: "nom", Samples: []string{"Alice Dupont", "Chloé Lambert"}, PersonNames: true},
{Lang: "fr", Table: "villes", Column: "nom", Samples: []string{"Namur", "Liège"}},
{Lang: "en", Table: "customers", Column: "name", PersonNames: true}, // no samples: missed
{Lang: "en", Table: "orders", Column: "name", Samples: []string{"Alice Dupont", "Bob Martin"}}, // labelled not, found
})
if got, want := *rep["all"], (confusion{TP: 1, FP: 1, FN: 1, TN: 1}); got != want {
t.Errorf("all: %+v, want %+v", got, want)
}
if m["namecolumns/fr/recall"] != 1 || m["namecolumns/fr/precision"] != 1 || m["namecolumns/en/recall"] != 0 ||
m["namecolumns/en/precision"] != 0 || m["namecolumns/all/recall"] != 0.5 {
t.Errorf("metrics %v", m)
}
}

// A detector weakened in a test (every iban match dropped) fails the
// bench, naming the iban recalls.
func TestBenchFailsOnWeakDetector(t *testing.T) {
Expand Down
166 changes: 166 additions & 0 deletions pii/internal/benchgen/gen/columns.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,166 @@
package gen

import (
"fmt"
"math/rand/v2"
"strings"
)

// Name columns: a table, a column whose name means "name" and a few
// sampled values, labelled person names or not. Positives are tables of
// people, or neutral tables, whose name column holds person names;
// negatives are tables of things, places and labels ("products.name",
// "roles.name", "cities.name", "tags.name") and tables of people whose
// name column holds no person names (companies, logins).

// ColumnCase is one sampled column, labelled.
type ColumnCase struct {
ID string `json:"id"`
Lang string `json:"lang"`
Table string `json:"table"`
Column string `json:"column"`
Samples []string `json:"samples"`
PersonNames bool `json:"person_names"`
}

// Sizes of the column blocks, per language.
const (
columnPositivesPerLang = 40
columnNegativesPerLang = 40
columnSamplesMin = 3
columnSamplesMax = 8
)

var nameColumns = map[string][]string{
"en": {"name", "full_name", "fullName", "customer_name", "display_name", "contact_name"},
"fr": {"nom", "nom_complet", "name", "nom_client", "full_name"},
"nl": {"naam", "volledige_naam", "name", "klant_naam", "naam_klant"},
"de": {"name", "vollstaendiger_name", "kunde_name", "full_name"},
}

var personTablesByLang = map[string][]string{
"en": {"customers", "users", "employees", "contacts", "clients", "public.customers", "crm.contacts"},
"fr": {"clients", "utilisateurs", "employes", "contacts", "personnes", "membres"},
"nl": {"klanten", "gebruikers", "medewerkers", "contacten", "leden", "personen"},
"de": {"kunden", "benutzer", "mitarbeiter", "kontakte", "mitglieder"},
}

var neutralTablesByLang = map[string][]string{
"en": {"orders", "invoices", "bookings", "registrations"},
"fr": {"commandes", "factures", "reservations", "inscriptions"},
"nl": {"bestellingen", "facturen", "reservaties", "inschrijvingen"},
"de": {"bestellungen", "rechnungen", "buchungen", "anmeldungen"},
}

// thingColumns are tables of things with the values of their name
// column.
var thingColumns = map[string][]struct {
table string
values []string
}{
"en": {
{"products", []string{"Laptop Pro 14", "USB-C Hub", "Wireless Mouse", "Standing Desk", "Office Chair", "Monitor 27 inch", "Paper A4"}},
{"roles", []string{"admin", "editor", "viewer", "owner", "billing", "support"}},
{"cities", []string{"London", "Manchester", "Leeds", "Bristol", "Namur", "Liège"}},
{"tags", []string{"urgent", "billing", "later", "follow-up", "vip", "spam"}},
{"categories", []string{"Fiction", "Comics", "Children", "Travel", "Cooking", "History"}},
{"companies", []string{"Acme Ltd", "Globex Corporation", "Initech", "Umbrella plc", "Stark Industries"}},
},
"fr": {
{"produits", []string{"Ordinateur portable 14", "Clé USB 32 Go", "Souris sans fil", "Bureau assis-debout", "Chaise de bureau"}},
{"roles", []string{"admin", "éditeur", "lecteur", "propriétaire", "comptabilité"}},
{"villes", []string{"Namur", "Liège", "Bruxelles", "Charleroi", "Mons", "Arlon"}},
{"tags", []string{"urgent", "facturation", "plus tard", "relance", "important"}},
{"categories", []string{"Romans", "Bandes dessinées", "Jeunesse", "Voyage", "Cuisine", "Histoire"}},
{"societes", []string{"Acme SA", "Durand et Fils SPRL", "Boulangerie du Centre", "Garage Moderne SRL"}},
},
"nl": {
{"producten", []string{"Laptop Pro 14", "USB-stick 32 GB", "Draadloze muis", "Zit-sta bureau", "Bureaustoel"}},
{"rollen", []string{"beheerder", "redacteur", "lezer", "eigenaar", "boekhouding"}},
{"steden", []string{"Gent", "Antwerpen", "Brugge", "Leuven", "Hasselt", "Mechelen"}},
{"tags", []string{"dringend", "facturatie", "later", "opvolgen", "belangrijk"}},
{"categorieen", []string{"Romans", "Strips", "Jeugd", "Reizen", "Koken", "Geschiedenis"}},
{"bedrijven", []string{"Acme NV", "Bakkerij De Molen BV", "Garage Centrum", "Drukkerij Vanhove"}},
},
"de": {
{"produkte", []string{"Laptop Pro 14", "USB-Stick 32 GB", "Kabellose Maus", "Stehschreibtisch", "Bürostuhl"}},
{"rollen", []string{"admin", "redakteur", "leser", "eigentümer", "buchhaltung"}},
{"staedte", []string{"Berlin", "Hamburg", "München", "Köln", "Aachen", "Eupen"}},
{"tags", []string{"dringend", "rechnung", "später", "nachfassen", "wichtig"}},
{"kategorien", []string{"Romane", "Comics", "Kinderbuch", "Reisen", "Kochen", "Geschichte"}},
{"firmen", []string{"Acme GmbH", "Bäckerei Müller KG", "Autohaus Zentrum", "Druckerei Weber AG"}},
},
}

// nonNames are name column values of a table of people that are no
// person's name: companies, logins, placeholders.
var nonNames = map[string][]string{
"en": {"Acme Ltd", "Globex Corporation", "admin", "jdoe42", "n/a", "unknown", "test account", "Support Team"},
"fr": {"Acme SA", "Garage Moderne SRL", "admin", "jdupont", "n/a", "inconnu", "compte test", "Service client"},
"nl": {"Acme NV", "Bakkerij De Molen BV", "admin", "jjanssens", "n.v.t.", "onbekend", "testaccount", "Klantendienst"},
"de": {"Acme GmbH", "Bäckerei Müller KG", "admin", "hmueller", "k. A.", "unbekannt", "Testkonto", "Kundendienst"},
}

// Columns returns the column cases of every language.
func Columns() []ColumnCase {
var out []ColumnCase
for _, lang := range Langs {
r := rng(lang + "/columns")
for i := range columnPositivesPerLang {
out = append(out, columnPositive(r, lang, fmt.Sprintf("c-%s-pos-%04d", lang, i+1)))
}
for i := range columnNegativesPerLang {
out = append(out, columnNegative(r, lang, fmt.Sprintf("c-%s-neg-%04d", lang, i+1)))
}
}
return out
}

func sampleCount(r *rand.Rand) int {
return columnSamplesMin + r.IntN(columnSamplesMax-columnSamplesMin+1)
}

// columnPositive is a name column of person names: in a table of people,
// a few values may be blank or no person's name; in a neutral table, all
// of them are names.
func columnPositive(r *rand.Rand, lang, id string) ColumnCase {
people := r.IntN(4) != 0
table := pick(r, neutralTablesByLang[lang])
if people {
table = pick(r, personTablesByLang[lang])
}
n := sampleCount(r)
samples := make([]string, n)
for i := range samples {
switch {
case people && i > 0 && r.IntN(6) == 0:
samples[i] = pick(r, append(nonNames[lang], ""))
case r.IntN(2) == 0:
samples[i] = knownName(r, lang)
default:
v, _ := personName(r, lang)
samples[i] = strings.Replace(v, unspanned, "", 1) // "Mme Dubois": a whole value
}
}
return ColumnCase{ID: id, Lang: lang, Table: table, Column: pick(r, nameColumns[lang]), Samples: samples, PersonNames: true}
}

// columnNegative is a name column of a table of things, or of a table of
// people holding no person's name.
func columnNegative(r *rand.Rand, lang, id string) ColumnCase {
n := sampleCount(r)
samples := make([]string, n)
if r.IntN(5) == 0 {
for i := range samples {
samples[i] = pick(r, nonNames[lang])
}
return ColumnCase{ID: id, Lang: lang, Table: pick(r, personTablesByLang[lang]), Column: pick(r, nameColumns[lang]),
Samples: samples}
}
t := thingColumns[lang][r.IntN(len(thingColumns[lang]))]
for i := range samples {
samples[i] = pick(r, t.values)
}
column := pick(r, []string{"name", "name", nameColumns[lang][0], "label_name"})
return ColumnCase{ID: id, Lang: lang, Table: t.table, Column: column, Samples: samples}
}
12 changes: 9 additions & 3 deletions pii/internal/benchgen/gen/gen.go
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
// Package gen builds the PII detection bench corpus: synthetic text cases
// in English, French, Dutch and German with the byte span and kind of
// every personal value they hold, hard negatives that look like personal
// data without being any, and field names labelled personal or
// quasi-identifying.
// data without being any, field names labelled personal or
// quasi-identifying, and sampled name columns labelled person names or
// not.
//
// Every value is generated: checksummed numbers have valid checksums in
// positives and invalid ones in hard negatives, e-mail addresses use the
Expand Down Expand Up @@ -254,7 +255,7 @@ func negative(r *rand.Rand, lang, id string) TextCase {
func pick(r *rand.Rand, xs []string) string { return xs[r.IntN(len(xs))] }

// Files returns every corpus file by name: one JSON Lines file per
// language and fields.jsonl.
// language, fields.jsonl and namecolumns.jsonl.
func Files() map[string][]byte {
out := map[string][]byte{}
for _, lang := range Langs {
Expand All @@ -269,6 +270,11 @@ func Files() map[string][]byte {
writeLine(&b, fc)
}
out["fields.jsonl"] = b.Bytes()
var c bytes.Buffer
for _, cc := range Columns() {
writeLine(&c, cc)
}
out["namecolumns.jsonl"] = c.Bytes()
return out
}

Expand Down
6 changes: 3 additions & 3 deletions pii/namecolumn.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ var thingTables = set(
"template", "setting", "plan", "status", "type", "service", "feature", "module", "page", "menu",
"produit", "categorie", "ville", "pays", "rue", "societe", "entreprise", "fichier",
"stad", "gemeente", "land", "straat", "bedrijf", "bestand", "groep",
"produkt", "kategorie", "stadt", "firma", "producto", "ciudad",
"produkt", "kategorie", "stadt", "staedte", "städte", "firma", "producto", "ciudad",
)

// Shares of the sampled values that must be person names, by table hint.
Expand Down Expand Up @@ -116,11 +116,11 @@ func tableHint(table string) int {
}

// singulars are a token and its singular candidates in en/fr/nl/de/es
// ("customers", "cities", "klanten", "leden").
// ("customers", "cities", "klanten", "leden", "mitglieder").
func singulars(t string) []string {
out := []string{t}
for _, suf := range []struct{ plural, single string }{
{"ies", "y"}, {"es", ""}, {"s", ""}, {"x", ""}, {"en", ""}, {"eden", "id"}, {"ten", "t"}, {"n", ""},
{"ies", "y"}, {"es", ""}, {"s", ""}, {"x", ""}, {"en", ""}, {"eden", "id"}, {"ten", "t"}, {"n", ""}, {"er", ""},
} {
if strings.HasSuffix(t, suf.plural) && len(t) > len(suf.plural) {
out = append(out, strings.TrimSuffix(t, suf.plural)+suf.single)
Expand Down
Loading
Loading