From 861c2360f90a55995203fa76c21cdd6bf361f047 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 25 Oct 2026 13:03:29 +0000 Subject: [PATCH 1/3] feat(matchers): match by phone number and email address PoiMatcher now also matches when the ATP feature and the OSM candidate share a descriptive phone number or email address, and their main categories are equal or a known pair (src/matchers/contact/ category_pairs.tsv). A brand:wikidata conflict still vetoes the match. Beyond 150 m, a contact match also needs the street address to agree. Restaurants, hotels, schools and fuel stations now get a matcher too, and more categories (cafes, fast food, bars, schools, clinics, doctors, fitness centres, museums, healthcare=*) get a match mask. The phone and email parsers move from pipeline::atp::nondescript to matchers::contact, so both sides are parsed alike. OSM numbers in national format go through the phonenumber crate, in the ATP feature's country. See docs/notes/2026-10-25-phone-and-email-matching.md. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01P43kACSyJVwszya8Z3d4qQ --- Cargo.lock | 150 ++++++++- Cargo.toml | 2 + docs/TECHNICAL_DESIGN.md | 15 +- .../2026-10-25-phone-and-email-matching.md | 156 +++++++++ scripts/README.md | 8 +- scripts/infer_contact_category_pairs.py | 152 +++++++++ scripts/test_infer_contact_category_pairs.py | 29 ++ src/matchers/address.rs | 112 +++++++ src/matchers/contact/category_pairs.tsv | 74 ++++ .../contact}/emails.rs | 10 +- src/matchers/contact/mod.rs | 317 ++++++++++++++++++ .../contact}/phones.rs | 6 +- src/matchers/mod.rs | 31 +- src/matchers/poi_matcher.rs | 259 +++++++++++++- src/pipeline/atp/mod.rs | 2 +- src/pipeline/atp/nondescript/mod.rs | 10 +- src/pipeline/conflate/mod.rs | 14 +- 17 files changed, 1298 insertions(+), 49 deletions(-) create mode 100644 docs/notes/2026-10-25-phone-and-email-matching.md create mode 100644 scripts/infer_contact_category_pairs.py create mode 100644 scripts/test_infer_contact_category_pairs.py create mode 100644 src/matchers/address.rs create mode 100644 src/matchers/contact/category_pairs.tsv rename src/{pipeline/atp/nondescript => matchers/contact}/emails.rs (95%) create mode 100644 src/matchers/contact/mod.rs rename src/{pipeline/atp/nondescript => matchers/contact}/phones.rs (95%) diff --git a/Cargo.lock b/Cargo.lock index 397296ab..d353d8d2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -265,7 +265,7 @@ dependencies = [ "memchr", "num-traits", "regex", - "regex-syntax", + "regex-syntax 0.8.11", ] [[package]] @@ -550,6 +550,15 @@ dependencies = [ "cc", ] +[[package]] +name = "cobs" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fa961b519f0b462e3a3b4a34b64d119eeaca1d59af726fe450bbba07a9fc0a1" +dependencies = [ + "thiserror 2.0.21", +] + [[package]] name = "codepage-437" version = "0.1.0" @@ -775,6 +784,18 @@ version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34" +[[package]] +name = "embedded-io" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef1a6892d9eef45c8fa6b9e0086428a2cca8491aca8f787c534a3d6d0bcb3ced" + +[[package]] +name = "embedded-io" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edd0f118536f44f5ccd48bcb8b111bdc3de888b58c74639dfb034a357d0f206d" + [[package]] name = "encode_unicode" version = "1.0.0" @@ -1233,6 +1254,7 @@ dependencies = [ "atomic-polyfill", "hash32 0.2.1", "rustc_version", + "serde", "spin", "stable_deref_trait", ] @@ -1748,6 +1770,12 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" +[[package]] +name = "linked-hash-map" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" + [[package]] name = "linux-raw-sys" version = "0.12.1" @@ -1781,6 +1809,15 @@ version = "0.18.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ef9ac18847474e638e3702b76c65d4eb93428471a74778ef0f1be711717f89b5" +[[package]] +name = "lru-cache" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31e24f1ad8321ca0e8a1e0ac13f23cb668e6f5466c2c57319f6a5cf1cc8e3b1c" +dependencies = [ + "linked-hash-map", +] + [[package]] name = "lru-slab" version = "0.1.3" @@ -1827,6 +1864,12 @@ dependencies = [ "libc", ] +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "miniz_oxide" version = "0.9.1" @@ -1878,6 +1921,16 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "normalize-line-endings" version = "0.3.0" @@ -1966,6 +2019,12 @@ version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +[[package]] +name = "oncemutex" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44d11de466f4a3006fe8a5e7ec84e93b79c70cb992ae0aa0eb631ad2df8abfe2" + [[package]] name = "openssl-probe" version = "0.2.1" @@ -2012,6 +2071,7 @@ dependencies = [ "osm_pbf_iter", "parquet", "parquet-geospatial", + "phonenumber", "piz", "predicates", "prost", @@ -2028,6 +2088,7 @@ dependencies = [ "tempfile", "time", "tokio", + "unicode-normalization", "uuid", "webpki-roots", "wkb", @@ -2097,6 +2158,26 @@ dependencies = [ "indexmap", ] +[[package]] +name = "phonenumber" +version = "0.3.10+9.0.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66d85d3cc5477bfe2f357939e7f02a5a566632508633af1a2c37577d0bef87d9" +dependencies = [ + "either", + "fnv", + "nom", + "once_cell", + "postcard", + "quick-xml", + "regex", + "regex-cache", + "serde", + "serde_derive", + "strum", + "thiserror 2.0.21", +] + [[package]] name = "pin-project-lite" version = "0.2.17" @@ -2131,6 +2212,19 @@ version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" +[[package]] +name = "postcard" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6764c3b5dd454e283a30e6dfe78e9b31096d9e32036b5d1eaac7a6119ccb9a24" +dependencies = [ + "cobs", + "embedded-io 0.4.0", + "embedded-io 0.6.1", + "heapless 0.7.17", + "serde", +] + [[package]] name = "potential_utf" version = "0.1.6" @@ -2442,7 +2536,7 @@ dependencies = [ "aho-corasick", "memchr", "regex-automata", - "regex-syntax", + "regex-syntax 0.8.11", ] [[package]] @@ -2453,9 +2547,27 @@ checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" dependencies = [ "aho-corasick", "memchr", - "regex-syntax", + "regex-syntax 0.8.11", ] +[[package]] +name = "regex-cache" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f7b62d69743b8b94f353b6b7c3deb4c5582828328bcb8d5fedf214373808793" +dependencies = [ + "lru-cache", + "oncemutex", + "regex", + "regex-syntax 0.6.29", +] + +[[package]] +name = "regex-syntax" +version = "0.6.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" + [[package]] name = "regex-syntax" version = "0.8.11" @@ -3020,6 +3132,27 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" +[[package]] +name = "strum" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf" +dependencies = [ + "strum_macros", +] + +[[package]] +name = "strum_macros" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7695ce3845ea4b33927c055a39dc438a45b059f7c1b3d91d38d10355fb8cbca7" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "subtle" version = "2.6.1" @@ -3097,7 +3230,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.3.4", + "getrandom 0.4.3", "once_cell", "rustix", "windows-sys 0.52.0", @@ -3387,6 +3520,15 @@ version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954" +[[package]] +name = "unicode-normalization" +version = "0.1.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75cd3fb03d5288b3ba9a1ee103ac7e9cbb59f0b03d64f8ddfcc04dd42758a85d" +dependencies = [ + "tinyvec", +] + [[package]] name = "unit-prefix" version = "0.5.2" diff --git a/Cargo.toml b/Cargo.toml index a56c0992..0dd201f5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,6 +31,7 @@ once_cell = "1.21.4" osm_pbf_iter = { version = ">=0.2.3", default-features = false } parquet = { version = "60.0.0", default-features = false, features = ["arrow", "geospatial", "zstd"] } parquet-geospatial = { version = "60.0.0", default-features = false } +phonenumber = "0.3.10" piz = "0.5.1" prost = "0.14.4" protobuf_iter = { version = "0.1.3", default-features = false } @@ -45,6 +46,7 @@ serde_json = { version = "1.0.150", default-features = false, features = ["std"] tempfile = "3.27.0" time = { version = "0.3.53", default-features = false, features = ["formatting", "parsing", "std"] } tokio = { version = "1.52.3", default-features = false } +unicode-normalization = "0.1.26" uuid = { version = "1.24.0", default-features = false, features = ["v4", "v5"] } webpki-roots = "1.0.6" wkb = "0.9.2" diff --git a/docs/TECHNICAL_DESIGN.md b/docs/TECHNICAL_DESIGN.md index fcc61f7e..010de0bd 100644 --- a/docs/TECHNICAL_DESIGN.md +++ b/docs/TECHNICAL_DESIGN.md @@ -346,7 +346,9 @@ fetch away. in spatial order; for each feature, builds a search cap around its centroid and queries `OsmFeatureIndex` for OSM candidates inside it, scoring each (see [`src/matchers/`](../src/matchers/)) and keeping - the best. Writes one row per ATP feature — matched or not — to + the best. Opens the nondescript phone and email sets from + `import_atp`, so that hotlines and central mailboxes never count as + matching evidence. Writes one row per ATP feature — matched or not — to `conflated.parquet`. See [`docs/outputs/CONFLATED_PARQUET.md`](outputs/CONFLATED_PARQUET.md) for that file’s schema. **~8 minutes** (~5 min matching, ~3 min @@ -603,10 +605,13 @@ until the full pipeline was proven out: - Nothing yet uploads suggested edits to any of the tools described above — that integration (most likely MapRoulette first, given its API is the closest fit) is future work, not yet designed in detail. -- The only matcher implemented, `PoiMatcher`, matches shop-category - POIs solely on an exact `brand:wikidata` tag. We’d like to extend - this well beyond it: matching stores by name too, not just - `brand:wikidata`, and matching well beyond stores (a tree matcher +- The only matcher implemented, `PoiMatcher`, matches shops, + restaurants, hotels, schools and fuel stations on an exact + `brand:wikidata` tag, or on a shared phone number or email address + with a compatible category (see + [`notes/2026-10-25-phone-and-email-matching.md`](notes/2026-10-25-phone-and-email-matching.md)). + We’d like to extend this well beyond it: matching by name too, and + matching well beyond points of interest (a tree matcher conflating municipal tree datasets against OSM by spatial distance and species looks particularly tractable). See [#708](https://github.com/brawer/osmdiffs/issues/708). diff --git a/docs/notes/2026-10-25-phone-and-email-matching.md b/docs/notes/2026-10-25-phone-and-email-matching.md new file mode 100644 index 00000000..c5be75e8 --- /dev/null +++ b/docs/notes/2026-10-25-phone-and-email-matching.md @@ -0,0 +1,156 @@ +# Matching by phone number and email address + +Investigation note behind the contact evidence in `PoiMatcher` +([`src/matchers/contact/`](../../src/matchers/contact/)). Before this +change, an AllThePlaces (ATP) feature could only match an OpenStreetMap +(OSM) feature with the same `brand:wikidata`. Most OSM shops have no +`brand:wikidata`, and many ATP spiders (schools, post offices, hotels +from tourism boards) have none either. The question was whether phone +numbers and email addresses can fill that gap, and at what precision. + +## Short answer + +Yes, with a category check. Phones and emails are less noisy than +names, but they identify a contact endpoint, which often serves +several features at one site: a flower shop inside a supermarket lists +the supermarket's number, a Western Union counter the number of the +shop it sits in, a hospital department the hospital's. The +[nondescript sets](2026-10-24-shared-phone-numbers.md) remove +hotlines and central mailboxes (values spread over 1 km or more), but +they keep same-site sharing on purpose. So a shared value counts only +when the two main categories are equal or a known pair, such as +`tourism=hotel` and `amenity=restaurant`. A `brand:wikidata` conflict +still vetoes the match, whatever the phone says. + +## How well the signals agree + +On Swiss `brand:wikidata` matches where both sides carry the value, +ATP and OSM agree on the phone 84% of the time, on email 91%, and on +the exact name only 70%. Most phone mismatches are branch vs. head +office numbers or stale OSM data, so a phone that differs is no reason +against a match. When phone and `brand:wikidata` both pick a feature, +they pick the same one in 92% of cases (email: 98%). The rest, in a +sample, were the mall or building, another business at the same site, +or a duplicate OSM node where only the duplicate had the phone. + +In Switzerland, about a fifth of OSM shops have a phone, whether or not +they have `brand:wikidata`. On the ATP side, 72% of rows with a +descriptive phone have no OSM feature anywhere with that number, so +recall is limited by OSM, not by the matcher. + +## The rule + +For each OSM candidate within 400 m: + +1. Both sides have `brand:wikidata` and share an ID: score 1.0 × + distance score, as before. +2. Both sides have `brand:wikidata` and share none: no match. +3. Otherwise, a descriptive phone or email agrees: 0.98 × category + factor × distance score. The category factor is 1.0 for equal main + categories, 0.95 for a pair in + [`category_pairs.tsv`](../../src/matchers/contact/category_pairs.tsv), + and 0 otherwise. Beyond 150 m, the OSM street and house number must + also appear in the ATP address. + +The main category is the first of `shop`, `amenity`, `tourism`, +`leisure`, `healthcare`, `craft` and `office` that a feature has. +Phone and fax keys are compared together, since ATP lists many +numbers as a phone on one feature and as a fax on another. + +ATP features of the shop, restaurant, lodging, school and fuel kinds +get a matcher if they have `brand:wikidata` or a descriptive phone or +email. Before, only shops with `brand:wikidata` did. The same change +adds cafes, fast food, bars, schools, colleges, doctors, clinics, car +rental, cinemas, money transfer, vets, fitness and sports centres, +museums and `healthcare=*` to the match masks. + +## Category pairs + +`scripts/infer_contact_category_pairs.py` reads a run without the +category check and counts the pairs of main categories among contact +matches. Names are the independent check: among contact matches with +equal categories, ATP and OSM share a name or brand word 83% of the +time. A pair is kept if its own rate is at least half of that, with at +least 20 name-agreeing matches. `florist` and `supermarket` share many +numbers but almost never a name, so that pair is dropped. + +The script kept 61 pairs from the planet. A hand check of samples +showed that 18 of them still mostly join two features at one site, so +they were removed: clinic or doctors and hospital, hospital and its +departments, money transfer and supermarkets or pharmacies (Western +Union counters), post office and supermarkets or copy shops, library +and museum, childcare or kindergarten and school (Zürich names each +after-school care after its school), clothes and department store, +cafe and fast food (McCafé), and fuel and convenience. The file lists +them as comments, with the reason. To update the list, rerun the +script on a fresh planet run and review the changes by hand. + +## Planet results + +ATP run `2026-10-10-13-32-41` against the planet of the same week, +2,097,628 ATP rows that the matcher looked at before this change. + +| Variant | Matched ATP rows | +|---|---:| +| `brand:wikidata`, shops only (before) | 825,901 | +| `brand:wikidata`, wider categories | 1,233,452 | +| `brand:wikidata` or phone and email, wider categories (this change) | PLANET_PR | + +No match from before was lost. To measure precision without hand +labels, a test run hid the brand tags on both sides and checked +whether contact evidence alone finds the OSM feature that +`brand:wikidata` found. Of about 183,000 such rows, it found the same +feature in 96.4% of cases with no category check, 97.6% with equal +categories only, and 97.1% with equal categories or a reviewed pair. + +The wider categories add 407,551 matches, mostly fast food (183k), +fuel stations (125k), cafes, restaurants and hotels. Their +`brand:wikidata` matches lie close: for charging stations, the median +distance is 8 m. + +Hand checks of new contact matches found about 95.6% right with the +inferred pairs, and about 97% after the review. The weakest group +before the review was ATP rows without `brand:wikidata` that matched +through a category pair: 60% right within 150 m and 33% beyond. After +the review, 37 of 40 sampled rows in that group were right. + +The largest gains are public and semi-public data: UK schools +(`gov_dfe_gias_gb`), USPS, NHS England, Poste Italiane, Slovenská +pošta, Chunghwa Post, Italian libraries (`iccu_it`), WorldCat and +`discover_swiss`. Among brand spiders, car dealers lead, since OSM +often maps a dealer by its own name without brand tags. Phone does +almost all the work; email alone adds little. + +## Phone parsing + +ATP writes phone numbers in international format, and so does most of +OSM. The matcher reads `+…` values with the same fast parser as the +nondescript sets, and only the rest with the +[`phonenumber`](https://crates.io/crates/phonenumber) crate, in the +ATP feature's `addr:country`. On the planet, that made 246,447 crate +calls at about 63 µs each, 15 CPU-seconds in all; the match step went +from 34 s to 35 s. Calling the crate for every value instead would make +13.3 million calls and slow the match step from 34 s to 190 s. + +National-format numbers are common in some countries. Among OSM phone +values near ATP features, they make up 38% in India, 25% in Thailand, +20% in Romania, 19.5% in Japan, 15.4% in South Korea and 8.9% in +Mexico, but under 2% in most of Europe and North America. In France +(1.3%), the crate can parse only 22% of them. + +## Caveats and follow-ups + +- Same-site errors remain where the categories are equal or paired: + a Shell station shop matched to the IGA supermarket next door, a + hotel matched to the hotel building's other hotel. One-to-one + resolution, where each OSM feature keeps only its best ATP row, + would catch some of these. +- A shared phone with disagreeing `brand:wikidata` is vetoed. Often + this is a rebrand where the number stayed (Audika took over KIND in + 2026; McDonald Nissan became Larry H. Miller Nissan). These would + make good candidates for human-reviewed brand updates. Regional + names are not rebrands, though: Fressnapf is Maxi Zoo in + French-speaking Switzerland. +- The precision figures come from small hand-labelled samples (about + 400 rows in all) and from the hide-the-wikidata test, which only + covers features that brand spiders know. diff --git a/scripts/README.md b/scripts/README.md index 63ff88a4..1b328f52 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -15,7 +15,13 @@ hardware. from a pinned commit of [`osm-testdata`](https://github.com/osmcode/osm-testdata). -Both are run by hand when someone notices upstream has moved; nothing +- [`infer_contact_category_pairs.py`](infer_contact_category_pairs.py): + suggests the pairs of categories in + `src/matchers/contact/category_pairs.tsv`, from a planet run without + the category check; the output needs a review by hand. See + [`../docs/notes/2026-10-25-phone-and-email-matching.md`](../docs/notes/2026-10-25-phone-and-email-matching.md). + +The first two are run by hand when someone notices upstream has moved; nothing here notifies you of a new release. Automating that is a low-priority, deliberately deferred feature request, tracked in [brawer/osmdiffs#555](https://github.com/brawer/osmdiffs/issues/555). diff --git a/scripts/infer_contact_category_pairs.py b/scripts/infer_contact_category_pairs.py new file mode 100644 index 00000000..5bb4388c --- /dev/null +++ b/scripts/infer_contact_category_pairs.py @@ -0,0 +1,152 @@ +# SPDX-FileCopyrightText: 2026 Sascha Brawer +# SPDX-License-Identifier: MIT + +"""Infers which category pairs may match through a phone or email. + +Reads `conflated.parquet` from a run with contact matching and no +category check (`OSMDIFFS_CONTACT_MATCH=both OSMDIFFS_CONTACT_CATEGORY=any`), +keeps the rows whose match rests on a shared phone number or email +address (not on `brand:wikidata`), and counts the pairs of main +categories, such as `tourism=hotel` in AllThePlaces and +`amenity=restaurant` in OpenStreetMap. + +A shared number alone doesn't tell the same business apart from a +neighbor at the same site: a flower shop inside a supermarket lists the +supermarket's number. Names do, mostly. So a pair is kept if, among its +contact matches, names agree about as often as among contact matches +whose categories are equal. `shop=florist` and `shop=supermarket` share +many numbers (Florissimo inside Migros) but almost never a name word, so +that pair is dropped. + +The conflated file must come from a run with the category check +switched off. The pipeline has no such switch; the experimental +branch `contact-matching` has one (`OSMDIFFS_CONTACT_MATCH=both +OSMDIFFS_CONTACT_CATEGORY=any OSMDIFFS_POI_MASKS=wide`). Review +the output by hand before checking it in: some pairs with agreeing +names still join two features at one site, such as a Western Union +counter and the supermarket it sits in. The checked-in file keeps +the removed pairs as comments, with the reason. + +Usage: + uv run infer_contact_category_pairs.py conflated.parquet \\ + --min-name-agree 20 > new_pairs.tsv 2> dropped.txt +""" + +import argparse +import collections +import re +import sys +import unicodedata + +import duckdb + +CATEGORY_KEYS = ["shop", "amenity", "tourism", "leisure", "healthcare", "craft", "office"] +PHONE_KEYS = {"phone", "fax", "mobile", "contact:phone", "contact:fax", "contact:mobile"} + + +def main_category(tags): + for k in CATEGORY_KEYS: + v = tags.get(k) + if v and v != "no": + return f"{k}={v}" + return None + + +def phones(tags): + out = set() + for k, v in tags.items(): + if k in PHONE_KEYS: + for part in v.split(";"): + part = part.strip() + if part.startswith("+"): + digits = re.sub(r"[ \-.() ]", "", part[1:]) + if digits.isdigit() and 7 <= len(digits) <= 15: + out.add(digits) + return out + + +def emails(tags): + out = set() + for k, v in tags.items(): + if k == "email" or k == "contact:pec" or k.endswith(":email"): + for part in re.split("[;,]", v): + part = part.strip().lower() + if part.startswith("mailto:"): + part = part[7:] + part = part.split("?")[0] + if "@" in part: + out.add(part) + return out + + +def name_words(tags): + s = " ".join(tags.get(k, "") for k in ("name", "brand")) + s = unicodedata.normalize("NFKD", s.lower()) + s = "".join(c for c in s if not unicodedata.combining(c)) + return {w for w in re.split(r"[^\w]+", s) if len(w) >= 4} + + +def wikidata(tags): + return {q.strip() for q in tags.get("brand:wikidata", "").split(";") if q.strip()} + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("conflated") + ap.add_argument("--min-name-agree", type=int, default=3, + help="minimum contact matches with agreeing names per pair") + ap.add_argument("--min-ratio", type=float, default=0.5, + help="minimum name agreement, relative to equal categories") + args = ap.parse_args() + + # Only rows where both sides have a phone or email key; at planet + # scale, this keeps the Python loop below small. + has_contact = ( + "len(list_filter(map_keys({}), lambda k: k IN ({}) OR k = 'email' " + "OR k = 'contact:pec' OR k LIKE '%:email')) > 0" + ) + keys = ", ".join(f"'{k}'" for k in sorted(PHONE_KEYS)) + rows = duckdb.sql( + "SELECT map_from_entries(map_entries(atp.tags)), map_from_entries(map_entries(osm.tags)) " + f"FROM read_parquet('{args.conflated}') WHERE osm IS NOT NULL " + f"AND {has_contact.format('atp.tags', keys)} AND {has_contact.format('osm.tags', keys)}" + ).fetchall() + + pairs = collections.Counter() + pairs_named = collections.Counter() + same = same_named = 0 + for atp, osm in rows: + atp, osm = dict(atp), dict(osm) + if wikidata(atp) & wikidata(osm): + continue + if not (phones(atp) & phones(osm) or emails(atp) & emails(osm)): + continue + a, o = main_category(atp), main_category(osm) + if not a or not o: + continue + named = bool(name_words(atp) & name_words(osm)) + if a == o: + same += 1 + same_named += named + continue + key = tuple(sorted((a, o))) + pairs[key] += 1 + pairs_named[key] += named + + base = same_named / same if same else 0.0 + print("# Pairs of main categories (key=value) that one business often has in") + print("# AllThePlaces and OpenStreetMap. A contact match (shared phone or email)") + print("# between them counts; see src/matchers/contact/mod.rs. Generated by") + print("# scripts/infer_contact_category_pairs.py from contact matches whose names") + print(f"# agree; with equal categories, names agree in {same_named}/{same} ({base:.0%}).") + print(f"# Kept: >= {args.min_name_agree} name agreements and >= {args.min_ratio:.0%} of that rate.") + print("# Columns: category, category, contact matches, of which names agree.") + for key, n in pairs.most_common(): + k = pairs_named[key] + keep = k >= args.min_name_agree and k / n >= args.min_ratio * base + line = f"{key[0]}\t{key[1]}\t{n}\t{k}" + print(line if keep else f"# dropped: {line}", file=sys.stdout if keep else sys.stderr) + + +if __name__ == "__main__": + main() diff --git a/scripts/test_infer_contact_category_pairs.py b/scripts/test_infer_contact_category_pairs.py new file mode 100644 index 00000000..32e2852c --- /dev/null +++ b/scripts/test_infer_contact_category_pairs.py @@ -0,0 +1,29 @@ +# SPDX-FileCopyrightText: 2026 Sascha Brawer +# SPDX-License-Identifier: MIT + +from infer_contact_category_pairs import emails, main_category, name_words, phones + + +def test_main_category(): + assert main_category({"name": "X", "amenity": "cafe", "shop": "bakery"}) == "shop=bakery" + assert main_category({"shop": "no", "amenity": "cafe"}) == "amenity=cafe" + assert main_category({"name": "X"}) is None + + +def test_phones(): + assert phones({"phone": "+41 44 417 77 77; +41 44 417 77 78"}) == {"41444177777", "41444177778"} + assert phones({"contact:fax": "+41 44 417 77 77"}) == {"41444177777"} + assert phones({"phone": "044 417 77 77"}) == set() + assert phones({"name": "+41 44 417 77 77"}) == set() + + +def test_emails(): + assert emails({"email": "Info@Example.ch, mailto:sales@example.ch?subject=x"}) == { + "info@example.ch", + "sales@example.ch", + } + assert emails({"contact:email": "x"}) == set() + + +def test_name_words(): + assert name_words({"name": "Zürich Hauptbahnhof", "brand": "SBB"}) == {"zurich", "hauptbahnhof"} diff --git a/src/matchers/address.rs b/src/matchers/address.rs new file mode 100644 index 00000000..0f292dcf --- /dev/null +++ b/src/matchers/address.rs @@ -0,0 +1,112 @@ +//! Cheap street address agreement check between AllThePlaces and +//! OpenStreetMap, used to confirm far-away match candidates. No +//! address parsing: OSM's split `addr:street`/`addr:housenumber` are +//! looked up in whatever address text the ATP feature carries. + +fn find<'a>(tags: &[(&'a str, &'a str)], key: &str) -> Option<&'a str> { + tags.iter().find(|(k, _)| *k == key).map(|(_, v)| *v) +} + +/// Case- and accent-folded alphanumeric tokens, split at +/// non-alphanumeric characters and at letter/digit boundaries, but +/// otherwise unsegmented: `Zürcherstr. 4` → `zurcherstr`, `4`; +/// `銀座4-6-16` → `銀座`, `4`, `6`, `16`. This needs no word +/// segmentation, so it works the same for scripts without spaces. +fn fold_tokens(s: &str) -> Vec { + use unicode_normalization::{UnicodeNormalization, char::is_combining_mark}; + let mut tokens = Vec::new(); + let mut cur = String::new(); + let mut cur_numeric = false; + for c in s + .nfkd() + .filter(|c| !is_combining_mark(*c)) + .flat_map(char::to_lowercase) + { + let c = if c == 'ß' { 's' } else { c }; + if !c.is_alphanumeric() { + if !cur.is_empty() { + tokens.push(std::mem::take(&mut cur)); + } + continue; + } + let numeric = c.is_numeric(); + if !cur.is_empty() && numeric != cur_numeric { + tokens.push(std::mem::take(&mut cur)); + } + cur_numeric = numeric; + cur.push(c); + } + if !cur.is_empty() { + tokens.push(cur); + } + tokens +} + +/// Format-agnostic check that needs no address parsing on the ATP +/// side: OSM's `addr:housenumber` tokens must appear in order in ATP's +/// address text (`349-B` in `Mythenquai 349b`), and OSM's +/// `addr:street` must appear in it with spaces and punctuation +/// removed, or start with one ATP word of 5+ letters (abbreviations +/// like `Zürcherstr.`). The ATP text is all of `addr:street_address`, +/// `addr:street`, `addr:housenumber` and `addr:full` that are present. +pub fn osm_address_in_atp_text(atp_tags: &[(&str, &str)], osm_tags: &[(&str, &str)]) -> bool { + let (Some(street), Some(hn)) = ( + find(osm_tags, "addr:street"), + find(osm_tags, "addr:housenumber"), + ) else { + return false; + }; + let text: Vec<&str> = [ + "addr:street_address", + "addr:street", + "addr:housenumber", + "addr:full", + ] + .iter() + .filter_map(|k| find(atp_tags, k)) + .collect(); + if text.is_empty() { + return false; + } + let atp = fold_tokens(&text.join(" ")); + let hn = fold_tokens(hn); + let street: String = fold_tokens(street).concat(); + if hn.is_empty() || street.is_empty() { + return false; + } + // OSM's house number tokens, in order, somewhere in ATP's text. + let hn_ok = atp.windows(hn.len()).any(|w| w == hn.as_slice()); + let compact = atp.concat(); + let street_ok = compact.contains(&street) + || atp.iter().any(|t| { + t.chars().count() >= 5 + && !t.chars().any(|c| c.is_ascii_digit()) + && street.starts_with(t.as_str()) + }); + hn_ok && street_ok +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn containment_check() { + let osm = [("addr:street", "Zürcherstrasse"), ("addr:housenumber", "4")]; + let yes = |t: &[(&str, &str)]| osm_address_in_atp_text(t, &osm); + assert!(yes(&[("addr:full", "Zürcherstrasse 4, 8640 Rapperswil")])); + assert!(yes(&[("addr:street_address", "ZURCHERSTRASSE 4")])); + assert!(yes(&[("addr:street_address", "Zürcherstr. 4")])); + assert!(yes(&[ + ("addr:street", "Zürcherstrasse"), + ("addr:housenumber", "4") + ])); + assert!(!yes(&[("addr:full", "Zürcherstrasse 6, 8640 Rapperswil")])); + assert!(!yes(&[("addr:full", "Bahnhofstrasse 4, 8640 Rapperswil")])); + let osm_b = [("addr:street", "Mythenquai"), ("addr:housenumber", "349-B")]; + assert!(osm_address_in_atp_text( + &[("addr:street_address", "Mythenquai 349b")], + &osm_b + )); + } +} diff --git a/src/matchers/contact/category_pairs.tsv b/src/matchers/contact/category_pairs.tsv new file mode 100644 index 00000000..dbc1931c --- /dev/null +++ b/src/matchers/contact/category_pairs.tsv @@ -0,0 +1,74 @@ +# Pairs of main categories (key=value) that one business often has in +# AllThePlaces and OpenStreetMap. A contact match (shared phone or email) +# between them counts; see src/matchers/contact/mod.rs. +# +# Generated by scripts/infer_contact_category_pairs.py from a planet run +# (ATP 2026-10-10, --min-name-agree 20): pairs whose contact matches agree +# on name about as often as contact matches with equal categories (83%). +# Then reviewed by hand against labelled samples; the pairs removed below +# mostly describe two features at one site that share a phone number. +# +# Columns: category, category, contact matches, of which names agree. +shop=car shop=car_repair 5548 5090 +shop=car shop=car_parts 654 608 +amenity=fast_food amenity=restaurant 473 391 +amenity=fuel shop=yes 386 307 +amenity=restaurant tourism=hotel 304 287 +amenity=car_rental shop=car 303 269 +shop=car_parts shop=car_repair 283 250 +shop=convenience shop=supermarket 265 211 +amenity=clinic amenity=doctors 174 164 +amenity=pharmacy shop=chemist 166 109 +tourism=hostel tourism=hotel 119 107 +shop=motorcycle shop=motorcycle_repair 92 81 +shop=car_repair shop=truck_repair 81 71 +amenity=car_rental shop=car_repair 79 50 +shop=doityourself shop=hardware 69 61 +amenity=college amenity=school 69 68 +healthcare=audiologist shop=hearing_aids 68 66 +shop=car_repair shop=tyres 67 51 +amenity=childcare amenity=kindergarten 56 54 +shop=car shop=motorcycle 49 40 +amenity=pub amenity=restaurant 45 43 +shop=bag shop=fashion_accessories 44 43 +shop=truck shop=truck_repair 41 31 +shop=charity shop=second_hand 40 37 +shop=car shop=truck_repair 39 29 +leisure=fitness_centre leisure=sports_centre 39 31 +shop=cosmetics shop=hairdresser_supply 37 35 +shop=doityourself shop=trade 36 32 +shop=beauty shop=cosmetics 35 29 +shop=furniture shop=interior_decoration 35 34 +healthcare=optometrist shop=optician 33 22 +shop=fashion_accessories shop=leather 31 29 +shop=hearing_aids shop=optician 31 21 +shop=car shop=truck 29 22 +tourism=guest_house tourism=hotel 29 26 +shop=car shop=motorcycle_repair 26 21 +shop=hardware shop=trade 26 21 +shop=outdoor shop=sports 26 20 +shop=car_repair shop=truck 25 21 +shop=interior_decoration shop=variety_store 25 23 +shop=rental shop=tool_hire 23 21 +shop=hardware shop=paint 22 21 +shop=baby_goods shop=clothes 21 20 + +# Removed by hand (same columns, then why): +# amenity=clinic amenity=hospital 310 186 a clinic or cancer centre inside a hospital campus, not the hospital +# amenity=money_transfer shop=supermarket 305 231 Western Union counter inside a shop +# amenity=childcare amenity=school 219 195 after-school care named after its school (Zurich) +# amenity=library tourism=museum 202 128 museum library vs. museum (iccu_it, worldcat) +# amenity=money_transfer shop=convenience 150 124 Western Union counter inside a shop +# amenity=kindergarten amenity=school 104 97 after-school care named after its school (Zurich) +# amenity=fuel shop=convenience 87 51 station shop mapped separately from the fuel station +# shop=clothes shop=department_store 64 27 concession inside a department store +# amenity=post_office shop=supermarket 53 36 postal counter inside a shop +# amenity=money_transfer amenity=pharmacy 51 45 Western Union counter inside a shop +# amenity=post_office shop=convenience 42 20 postal counter inside a shop +# amenity=cafe amenity=fast_food 41 27 McCafe inside McDonald's +# amenity=doctors amenity=hospital 41 33 same as clinic/hospital +# amenity=money_transfer shop=chemist 28 25 Western Union counter inside a shop +# amenity=post_office shop=copyshop 22 21 postal counter inside a shop +# amenity=hospital healthcare=birthing_centre 22 22 hospital department +# amenity=hospital healthcare=rehabilitation 21 20 hospital department +# amenity=hospital healthcare=laboratory 21 20 hospital department diff --git a/src/pipeline/atp/nondescript/emails.rs b/src/matchers/contact/emails.rs similarity index 95% rename from src/pipeline/atp/nondescript/emails.rs rename to src/matchers/contact/emails.rs index 94ff9c2d..50a6408a 100644 --- a/src/pipeline/atp/nondescript/emails.rs +++ b/src/matchers/contact/emails.rs @@ -1,9 +1,9 @@ -//! Email addresses, for [super::Signal::Email]. +//! Email addresses in tag values. /// Whether `key` is a tag key that may hold email addresses: `email`, /// `contact:email`, department-specific ones like `sales:email` or /// `repair:email`, and `contact:pec` (Italian certified email). -pub fn is_email_key(key: &str) -> bool { +pub(crate) fn is_email_key(key: &str) -> bool { key == "email" || key == "contact:pec" || key.ends_with(":email") } @@ -12,7 +12,7 @@ pub fn is_email_key(key: &str) -> bool { /// some AllThePlaces spiders), normalized with [normalize_email], and /// hashed with [email_hash]. Parts that aren't a plausible address are /// skipped. -pub fn parse_emails(value: &str) -> impl Iterator + '_ { +pub(crate) fn parse_emails(value: &str) -> impl Iterator + '_ { value .split([';', ',']) .filter_map(normalize_email) @@ -25,7 +25,7 @@ pub fn parse_emails(value: &str) -> impl Iterator + '_ { /// mail system treats it so in practice. Returns `None` for anything /// that doesn't look like an address (one `@`, a non-empty local part /// without spaces, a domain with at least two labels). -pub fn normalize_email(part: &str) -> Option { +pub(crate) fn normalize_email(part: &str) -> Option { let mut s = part.trim(); s = s .strip_prefix('<') @@ -67,7 +67,7 @@ pub fn normalize_email(part: &str) -> Option { /// version can be read by another. With about 430,000 distinct /// addresses in ATP, the chance of any collision is about 1 in 200 /// million; a collision would only make one more address nondescript. -pub fn email_hash(normalized: &str) -> u64 { +pub(crate) fn email_hash(normalized: &str) -> u64 { let digest = aws_lc_rs::digest::digest(&aws_lc_rs::digest::SHA256, normalized.as_bytes()); let mut bytes = [0_u8; 8]; bytes.copy_from_slice(&digest.as_ref()[..8]); diff --git a/src/matchers/contact/mod.rs b/src/matchers/contact/mod.rs new file mode 100644 index 00000000..9e054812 --- /dev/null +++ b/src/matchers/contact/mod.rs @@ -0,0 +1,317 @@ +//! Phone numbers and email addresses as matching evidence. +//! +//! A phone number or email address that an AllThePlaces feature and an +//! OpenStreetMap candidate both carry is good evidence that they +//! describe the same *site*, but not necessarily the same *feature*: a +//! flower shop inside a supermarket often lists the supermarket's +//! number, and a hotel bar the hotel's. So a contact match also needs +//! compatible main categories, see [category_factor]. Values that +//! AllThePlaces lists for places far apart (hotlines, central +//! mailboxes) are nondescript and never count; see [Nondescript] and +//! `pipeline::atp::nondescript`. For the background, see +//! `docs/notes/2026-10-25-phone-and-email-matching.md`. + +use crate::tables::U64Set; +use anyhow::{Context, Result}; +use std::collections::HashSet; +use std::path::Path; +use std::str::FromStr; +use std::sync::LazyLock; + +mod emails; +mod phones; + +#[cfg(test)] +pub(crate) use emails::email_hash; +pub(crate) use emails::{is_email_key, parse_emails}; +pub(crate) use phones::{is_phone_key, parse_phone_numbers}; + +/// Score factor of a contact match whose main categories are equal. +/// Below 1.0, so a `brand:wikidata` match at the same distance wins. +pub const CONTACT_FACTOR: f64 = 0.98; + +/// Extra factor for a contact match whose main categories are a known +/// pair, see [category_factor]. +pub const CATEGORY_PAIR_FACTOR: f64 = 0.95; + +/// Phone numbers and email addresses that AllThePlaces lists for +/// places far apart, as written by `import_atp`. A shared value in one +/// of these sets is no evidence for a match. +pub struct Nondescript { + phones: Option, + emails: Option, +} + +impl Nondescript { + /// Opens the two sets written by `import_atp`. + pub fn open(phones: &Path, emails: &Path) -> Result { + let open = |path: &Path| { + U64Set::open(path).with_context(|| format!("could not open {}", path.display())) + }; + Ok(Nondescript { + phones: Some(open(phones)?), + emails: Some(open(emails)?), + }) + } + + /// No nondescript values at all, for tests. + #[cfg(test)] + pub fn none() -> Nondescript { + Nondescript { + phones: None, + emails: None, + } + } + + fn is_phone(&self, number: u64) -> bool { + self.phones.as_ref().is_some_and(|s| s.contains(number)) + } + + fn is_email(&self, hash: u64) -> bool { + self.emails.as_ref().is_some_and(|s| s.contains(hash)) + } +} + +/// The descriptive phone numbers and email addresses of an +/// AllThePlaces feature. +#[derive(Debug, Default)] +pub struct AtpContacts { + /// Sorted, as encoded by [parse_phone_numbers]. + phones: Vec, + /// Sorted, as encoded by [parse_emails]. + emails: Vec, + /// From `addr:country`, for parsing OSM numbers in national format. + country: Option, +} + +impl AtpContacts { + pub fn from_tags<'a>( + tags: impl Iterator, + nondescript: &Nondescript, + ) -> AtpContacts { + let mut c = AtpContacts::default(); + for (k, v) in tags { + if is_phone_key(k) { + c.phones + .extend(parse_phone_numbers(v).filter(|n| !nondescript.is_phone(*n))); + } else if is_email_key(k) { + c.emails + .extend(parse_emails(v).filter(|h| !nondescript.is_email(*h))); + } else if k == "addr:country" { + c.country = phonenumber::country::Id::from_str(v.trim()).ok(); + } + } + c.phones.sort_unstable(); + c.phones.dedup(); + c.emails.sort_unstable(); + c.emails.dedup(); + c + } + + pub fn is_empty(&self) -> bool { + self.phones.is_empty() && self.emails.is_empty() + } + + /// Whether `osm_tags` carry one of our phone numbers or emails. + pub fn agrees<'a>(&self, osm_tags: impl Iterator) -> bool { + for (k, v) in osm_tags { + let hit = if is_phone_key(k) { + !self.phones.is_empty() + && v.split(';') + .filter_map(|part| parse_osm_phone(part, self.country)) + .any(|n| self.phones.binary_search(&n).is_ok()) + } else if is_email_key(k) { + !self.emails.is_empty() + && parse_emails(v).any(|h| self.emails.binary_search(&h).is_ok()) + } else { + false + }; + if hit { + return true; + } + } + false + } +} + +/// Parses one OSM phone value. Most are in international format, which +/// [parse_phone_numbers] reads quickly. The rest, such as `044 417 77 +/// 77`, go through the `phonenumber` crate in the AllThePlaces +/// feature's country. That is slow (about 60 µs a call) but rare: on +/// the planet, it adds about one second in all. In some countries, +/// such as India, Thailand or Japan, a fifth or more of OSM's numbers +/// are in national format. +fn parse_osm_phone(part: &str, country: Option) -> Option { + let part = part.trim(); + if part.starts_with('+') { + return parse_phone_numbers(part).next(); + } + let number = phonenumber::parse(country, part).ok()?; + if !phonenumber::is_valid(&number) { + return None; + } + let e164 = number.format().mode(phonenumber::Mode::E164).to_string(); + parse_phone_numbers(&e164).next() +} + +/// Keys whose value is a feature's main category, in order of priority. +const CATEGORY_KEYS: &[&str] = &[ + "shop", + "amenity", + "tourism", + "leisure", + "healthcare", + "craft", + "office", +]; + +/// A feature's main category as `key=value`, such as `shop=florist`: +/// the value of the first key in [CATEGORY_KEYS] that the feature has. +pub fn main_category<'a>(tags: impl Iterator) -> Option { + let mut best: Option<(usize, &str, &str)> = None; + for (k, v) in tags { + if let Some(i) = CATEGORY_KEYS.iter().position(|c| *c == k) + && best.is_none_or(|(j, _, _)| i < j) + && v != "no" + { + best = Some((i, k, v)); + } + } + best.map(|(_, k, v)| format!("{k}={v}")) +} + +/// Pairs of main categories that one business often has in AllThePlaces +/// and OpenStreetMap, such as `tourism=hotel` and `amenity=restaurant`. +/// Inferred from a planet run by `scripts/infer_contact_category_pairs.py` +/// and then reviewed by hand; see the file's header. +static CATEGORY_PAIRS: LazyLock> = LazyLock::new(|| { + let mut pairs = HashSet::new(); + for line in include_str!("category_pairs.tsv").lines() { + let line = line.trim(); + if line.is_empty() || line.starts_with('#') { + continue; + } + let mut cols = line.split('\t'); + if let (Some(a), Some(b)) = (cols.next(), cols.next()) { + pairs.insert((a.to_string(), b.to_string())); + pairs.insert((b.to_string(), a.to_string())); + } + } + pairs +}); + +/// How well two main categories agree, for a contact match: 1.0 if +/// they are equal, [CATEGORY_PAIR_FACTOR] if they are a known pair, +/// and 0.0 otherwise, also when either side has no main category. +pub fn category_factor(atp: Option<&str>, osm: Option<&str>) -> f64 { + match (atp, osm) { + (Some(a), Some(o)) if a == o => 1.0, + (Some(a), Some(o)) if CATEGORY_PAIRS.contains(&(a.to_string(), o.to_string())) => { + CATEGORY_PAIR_FACTOR + } + _ => 0.0, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::TempDir; + + #[test] + fn test_main_category() { + let t = [("name", "X"), ("amenity", "cafe"), ("shop", "bakery")]; + assert_eq!(main_category(t.into_iter()).as_deref(), Some("shop=bakery")); + let t = [("shop", "no"), ("amenity", "cafe")]; + assert_eq!( + main_category(t.into_iter()).as_deref(), + Some("amenity=cafe") + ); + assert_eq!(main_category([("name", "X")].into_iter()), None); + } + + #[test] + fn test_category_factor() { + let f = category_factor; + assert_eq!(f(Some("shop=car"), Some("shop=car")), 1.0); + assert_eq!( + f(Some("shop=car"), Some("shop=car_repair")), + CATEGORY_PAIR_FACTOR + ); + assert_eq!( + f(Some("shop=car_repair"), Some("shop=car")), + CATEGORY_PAIR_FACTOR + ); + // Removed by hand: a Western Union counter inside a supermarket. + assert_eq!( + f(Some("amenity=money_transfer"), Some("shop=supermarket")), + 0.0 + ); + assert_eq!(f(Some("shop=florist"), Some("shop=supermarket")), 0.0); + assert_eq!(f(None, Some("shop=car")), 0.0); + assert_eq!(f(None, None), 0.0); + } + + #[test] + fn test_category_pairs_file() { + for (a, b) in CATEGORY_PAIRS.iter() { + assert!(a.contains('='), "{a}"); + assert!(b.contains('='), "{b}"); + assert_ne!(a, b); + } + assert!(CATEGORY_PAIRS.len() >= 2 * 40); + } + + #[test] + fn test_parse_osm_phone() { + let ch = Some(phonenumber::country::Id::CH); + assert_eq!(parse_osm_phone("044 417 77 77", ch), Some(41444177777)); + assert_eq!( + parse_osm_phone(" +41 44 417 77 77", None), + Some(41444177777) + ); + assert_eq!(parse_osm_phone("12", ch), None); + assert_eq!(parse_osm_phone("044 417 77 77", None), None); + } + + #[test] + fn test_agrees() { + let atp = [ + ("addr:country", "CH"), + ("phone", "+41 44 417 77 77"), + ("email", "Info@Example.ch"), + ]; + let c = AtpContacts::from_tags(atp.into_iter(), &Nondescript::none()); + assert!(c.agrees([("phone", "+41 44 417 77 77")].into_iter())); + assert!(c.agrees([("contact:phone", "044 417 77 77")].into_iter())); + assert!(c.agrees([("fax", "+41 1 234 56 78; +41 44 417 77 77")].into_iter())); + assert!(c.agrees([("contact:email", "info@example.ch")].into_iter())); + assert!(!c.agrees([("phone", "+41 44 417 77 78")].into_iter())); + assert!(!c.agrees([("name", "+41 44 417 77 77")].into_iter())); + assert!(!c.agrees([("email", "sales@example.ch")].into_iter())); + } + + #[test] + fn test_nondescript() -> Result<()> { + let dir = TempDir::new()?; + let phones = dir.path().join("phones"); + let emails = dir.path().join("emails"); + U64Set::create([41444177777].into_iter(), dir.path(), &phones, 1 << 20)?; + U64Set::create( + [email_hash("info@example.ch")].into_iter(), + dir.path(), + &emails, + 1 << 20, + )?; + let nondescript = Nondescript::open(&phones, &emails)?; + let atp = [("phone", "+41 44 417 77 77"), ("email", "info@example.ch")]; + let c = AtpContacts::from_tags(atp.into_iter(), &nondescript); + assert!(c.is_empty()); + + let atp = [("phone", "+41 44 417 77 77; +41 55 210 63 10")]; + let c = AtpContacts::from_tags(atp.into_iter(), &nondescript); + assert_eq!(c.phones, [41552106310]); + assert!(!c.agrees([("phone", "+41 44 417 77 77")].into_iter())); + Ok(()) + } +} diff --git a/src/pipeline/atp/nondescript/phones.rs b/src/matchers/contact/phones.rs similarity index 95% rename from src/pipeline/atp/nondescript/phones.rs rename to src/matchers/contact/phones.rs index c4978bb8..ec6c8f72 100644 --- a/src/pipeline/atp/nondescript/phones.rs +++ b/src/matchers/contact/phones.rs @@ -1,4 +1,4 @@ -//! Phone and fax numbers, for [super::Signal::Phone]. +//! Phone and fax numbers in tag values. /// Tag keys whose values may hold phone or fax numbers. Phone and fax /// share one key space: in ATP, thousands of numbers show up as a phone @@ -19,7 +19,7 @@ const MIN_DIGITS: usize = 7; const MAX_DIGITS: usize = 15; /// Whether `key` is a tag key that may hold phone or fax numbers. -pub fn is_phone_key(key: &str) -> bool { +pub(crate) fn is_phone_key(key: &str) -> bool { PHONE_KEYS.contains(&key) } @@ -32,7 +32,7 @@ pub fn is_phone_key(key: &str) -> bool { /// Parts that aren't in international format -- a few spiders emit /// local numbers, or junk like `9350-0` or `1800 PARKING` -- are /// skipped. -pub fn parse_phone_numbers(value: &str) -> impl Iterator + '_ { +pub(crate) fn parse_phone_numbers(value: &str) -> impl Iterator + '_ { value.split(';').filter_map(parse_phone_number) } diff --git a/src/matchers/mod.rs b/src/matchers/mod.rs index 03063dbe..b6e4c46d 100644 --- a/src/matchers/mod.rs +++ b/src/matchers/mod.rs @@ -10,6 +10,8 @@ use serde::{Deserialize, Serialize}; use std::sync::LazyLock; use wkb::reader::read_wkb; +mod address; +pub mod contact; mod poi_matcher; /// Mean Earth radius, in meters -- used throughout this module to turn @@ -70,6 +72,8 @@ impl MatchMask { self.0 |= match (key, value) { ("amenity", "arts_centre") => Self::SHOP.0, ("amenity", "bank") => Self::SHOP.0, + ("amenity", "bar") => Self::RESTAURANT.0, + ("amenity", "biergarten") => Self::RESTAURANT.0, ("amenity", "bench") => Self::STREET_FURNITURE.0, // Bike-share docking stations and drop-off points sit close // together and are often all named after their operator, so @@ -79,12 +83,20 @@ impl MatchMask { ("amenity", "bbq") => Self::STREET_FURNITURE.0, ("amenity", "bicycle_repair_station") => Self::STREET_FURNITURE.0, ("amenity", "bureau_de_change") => Self::SHOP.0, + ("amenity", "cafe") => Self::RESTAURANT.0, + ("amenity", "car_rental") => Self::SHOP.0, + ("amenity", "cinema") => Self::SHOP.0, + ("amenity", "clinic") => Self::SHOP.0, + ("amenity", "college") => Self::SCHOOL.0, ("amenity", "childcare") => Self::SCHOOL.0, ("amenity", "dentist") => Self::SHOP.0, + ("amenity", "doctors") => Self::SHOP.0, ("amenity", "drinking_water") => Self::STREET_FURNITURE.0, ("amenity", "driving_school") => Self::SCHOOL.0, ("amenity", "charging_station") => Self::FUEL.0, ("amenity", "ferry_terminal") => Self::TRANSIT.0, + ("amenity", "fast_food") => Self::RESTAURANT.0, + ("amenity", "food_court") => Self::RESTAURANT.0, ("amenity", "fountain") => Self::STREET_FURNITURE.0, ("amenity", "fuel") => Self::FUEL.0, ("amenity", "grit_bin") => Self::STREET_FURNITURE.0, @@ -92,6 +104,7 @@ impl MatchMask { ("amenity", "kindergarten") => Self::SCHOOL.0, ("amenity", "letter_box") => Self::STREET_FURNITURE.0, ("amenity", "library") => Self::SHOP.0, + ("amenity", "money_transfer") => Self::SHOP.0, ("amenity", "motorcycle_parking") => Self::PARKING.0, ("amenity", "nightclub") => Self::SHOP.0, ("amenity", "parking") => Self::PARKING.0, @@ -108,6 +121,8 @@ impl MatchMask { ("amenity", "theatre") => Self::SHOP.0, ("amenity", "trolley_bay") => Self::STREET_FURNITURE.0, ("amenity", "restaurant") => Self::RESTAURANT.0, + ("amenity", "school") => Self::SCHOOL.0, + ("amenity", "veterinary") => Self::SHOP.0, ("amenity", "university") => Self::SCHOOL.0, ("amenity", "vending_machine") => Self::STREET_FURNITURE.0, ("amenity", "waste_basket") => Self::STREET_FURNITURE.0, @@ -121,6 +136,9 @@ impl MatchMask { ("bicycle_rental", "shop") => Self::SHOP.0, + ("healthcare", "no") => 0, + ("healthcare", _) => Self::SHOP.0, + ("emergency", "acess_point") => Self::STREET_FURNITURE.0, ("emergency", "assembly_point") => Self::STREET_FURNITURE.0, ("emergency", "bleed_control_kit") => Self::STREET_FURNITURE.0, @@ -153,6 +171,8 @@ impl MatchMask { ("leisure", "bird_hide") => Self::STREET_FURNITURE.0, ("leisure", "bowling_alley") => Self::SHOP.0, ("leisure", "dance") => Self::SHOP.0, + ("leisure", "fitness_centre") => Self::SHOP.0, + ("leisure", "sports_centre") => Self::SHOP.0, ("leisure", "minigolf") => Self::SHOP.0, ("leisure", "firepit") => Self::STREET_FURNITURE.0, ("leisure", "hot_tub") => Self::STREET_FURNITURE.0, @@ -205,6 +225,7 @@ impl MatchMask { ("tourism", "hotel") => Self::LODGING.0, ("tourism", "hotel;motel") => Self::LODGING.0, ("tourism", "motel") => Self::LODGING.0, + ("tourism", "museum") => Self::SHOP.0, ("tourism", "wilderness_hut") => Self::LODGING.0, ("tower", _) => Self::STREET_FURNITURE.0, @@ -302,9 +323,13 @@ pub trait Matcher { /// For example, a matcher for brand stores will look at the brand /// of OpenStreetMap features, and only match when the candidate /// is of the same brand; a matcher for trees might consider -/// the plant species. -pub fn create_matcher(place: &Place) -> Option> { - if let Some(matcher) = poi_matcher::PoiMatcher::for_place(place) { +/// the plant species. `nondescript` holds the phone numbers and email +/// addresses that say nothing about which place a feature is. +pub fn create_matcher<'a>( + place: &'a Place, + nondescript: &contact::Nondescript, +) -> Option> { + if let Some(matcher) = poi_matcher::PoiMatcher::for_place(place, nondescript) { Some(Box::new(matcher)) } else { // TODO: Implement matchers for fire hydrants, defibrillators, etc. diff --git a/src/matchers/poi_matcher.rs b/src/matchers/poi_matcher.rs index 733a333b..fcaa9b31 100644 --- a/src/matchers/poi_matcher.rs +++ b/src/matchers/poi_matcher.rs @@ -1,4 +1,8 @@ -use super::{MatchMask, Matcher, OsmCandidate, geo_point_distance_score, parse_wikidata_ids}; +use super::contact::{AtpContacts, CONTACT_FACTOR, Nondescript, category_factor, main_category}; +use super::{ + MatchMask, Matcher, OsmCandidate, address, geo_point_distance, geo_point_distance_score, + parse_wikidata_ids, +}; use crate::places::Place; use geo::Centroid; use s2::{cell::Cell, cellid::CellID, point::Point}; @@ -18,39 +22,114 @@ fn find_brand_wikidata<'a>( .flat_map(|(_, v)| parse_wikidata_ids(v)) } +/// AllThePlaces features of these kinds get a [PoiMatcher]. +const POI_MASK: MatchMask = MatchMask( + MatchMask::SHOP.0 + | MatchMask::RESTAURANT.0 + | MatchMask::LODGING.0 + | MatchMask::SCHOOL.0 + | MatchMask::FUEL.0, +); + +/// Beyond this distance, a contact match needs the OSM street address +/// to appear in the ATP address, see [address::osm_address_in_atp_text]. +/// On the planet, only a third of the far contact matches through a +/// category pair were right without this check. +const CONTACT_ADDRESS_CHECK_BEYOND_METERS: f64 = 150.0; + +/// Matches points of interest, such as shops, restaurants, hotels and +/// schools, by evidence: +/// +/// - Equal `brand:wikidata`: the distance score. +/// - Both sides have `brand:wikidata`, but share no ID: no match, +/// whatever else agrees. +/// - Otherwise, a shared descriptive phone number or email address: +/// [CONTACT_FACTOR] times [category_factor] times the distance score. +/// Beyond [CONTACT_ADDRESS_CHECK_BEYOND_METERS], the street address +/// must agree too. +/// +/// Phones or emails that differ never lower a score: about one in six +/// right `brand:wikidata` matches has different numbers on the two +/// sides. pub struct PoiMatcher { center: Point, /// Usually a single ID; see [find_brand_wikidata]. brand_wikidata: Vec, + contacts: AtpContacts, + /// Main category as `key=value`, for contact matches. + category: Option, + /// The ATP feature's `addr:*` tags, for far contact matches. Empty + /// if `contacts` is empty. + address: Vec<(String, String)>, } impl PoiMatcher { - // TODO: For now, we only match based on brand:wikidata. - // We should also look at names, by computing the token sort ratio, + // TODO: We should also look at names, by computing the token sort ratio, // but we should do this in a way that works for CJK. So we need // a decent segmenter that works for CJK languages. Maybe Lindera? - pub fn for_place(place: &Place) -> Option { - if !place.mask.intersects(&MatchMask::SHOP) { + pub fn for_place(place: &Place, nondescript: &Nondescript) -> Option { + if !place.mask.intersects(&POI_MASK) { return None; } - let tags = place.tags.iter().map(|(k, v)| (k.as_str(), v.as_str())); - let brand_wikidata: Vec = find_brand_wikidata(tags).collect(); - if brand_wikidata.is_empty() { + let tags = || place.tags.iter().map(|(k, v)| (k.as_str(), v.as_str())); + let brand_wikidata: Vec = find_brand_wikidata(tags()).collect(); + let contacts = AtpContacts::from_tags(tags(), nondescript); + if brand_wikidata.is_empty() && contacts.is_empty() { return None; } - let center = Cell::from(CellID(place.s2_cell_id)).center(); + let (category, address) = if contacts.is_empty() { + (None, Vec::new()) + } else { + let address = tags() + .filter(|(k, _)| k.starts_with("addr:")) + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect(); + (main_category(tags()), address) + }; Some(PoiMatcher { - center, + center: Cell::from(CellID(place.s2_cell_id)).center(), brand_wikidata, + contacts, + category, + address, }) } + + /// The evidence factor for `candidate`, before distance: 1.0 for + /// equal `brand:wikidata`, below 1.0 for a contact match, 0.0 for + /// no match. + fn factor(&self, candidate: &OsmCandidate) -> f64 { + if !self.brand_wikidata.is_empty() { + let mut osm = find_brand_wikidata(candidate.tags()).peekable(); + if osm.peek().is_some() { + let shared = osm.any(|id| self.brand_wikidata.contains(&id)); + return if shared { 1.0 } else { 0.0 }; + } + } + if self.contacts.is_empty() || !self.contacts.agrees(candidate.tags()) { + return 0.0; + } + let osm_category = main_category(candidate.tags()); + CONTACT_FACTOR * category_factor(self.category.as_deref(), osm_category.as_deref()) + } + + fn address_agrees(&self, candidate: &OsmCandidate) -> bool { + let atp: Vec<(&str, &str)> = self + .address + .iter() + .map(|(k, v)| (k.as_str(), v.as_str())) + .collect(); + let osm: Vec<(&str, &str)> = candidate.tags().collect(); + address::osm_address_in_atp_text(&atp, &osm) + } } impl Matcher for PoiMatcher { fn score_osm_candidate(&self, candidate: &OsmCandidate) -> f64 { // Cheap check first: only decode the (potentially large) geometry - // for candidates that actually match on brand. - if !find_brand_wikidata(candidate.tags()).any(|id| self.brand_wikidata.contains(&id)) { + // for candidates that have some evidence. + let factor = self.factor(candidate); + if factor == 0.0 { return 0.0; } let Result::Ok(geometry) = candidate.geometry() else { @@ -59,7 +138,13 @@ impl Matcher for PoiMatcher { let Some(centroid) = geometry.centroid() else { return 0.0; }; - geo_point_distance_score(&self.center, ¢roid, 400.0) + if factor < 1.0 + && geo_point_distance(&self.center, ¢roid) > CONTACT_ADDRESS_CHECK_BEYOND_METERS + && !self.address_agrees(candidate) + { + return 0.0; + } + factor * geo_point_distance_score(&self.center, ¢roid, 400.0) } } @@ -251,9 +336,10 @@ mod tests { strings: &pool, }; - let ch_clothes_matcher = - PoiMatcher::for_place(&CH_CLOTHES_ATP).expect("should create matcher"); - let ch_kiosk_matcher = PoiMatcher::for_place(&CH_KIOSK_ATP).expect("should create matcher"); + let ch_clothes_matcher = PoiMatcher::for_place(&CH_CLOTHES_ATP, &Nondescript::none()) + .expect("should create matcher"); + let ch_kiosk_matcher = PoiMatcher::for_place(&CH_KIOSK_ATP, &Nondescript::none()) + .expect("should create matcher"); assert!(ch_clothes_matcher.score_osm_candidate(&clothes_candidate) > 0.5); assert!(ch_clothes_matcher.score_osm_candidate(&kiosk_candidate) == 0.0); assert!(ch_kiosk_matcher.score_osm_candidate(&clothes_candidate) == 0.0); @@ -287,4 +373,145 @@ mod tests { }; assert_eq!(ch_kiosk_matcher.score_osm_candidate(&other_candidate), 0.0); } + + /// Phone evidence for an ATP feature without `brand:wikidata`: a + /// shared number matches when the categories agree, never against a + /// `brand:wikidata` conflict, and never with a different category. + #[test] + fn test_poi_matcher_contact() { + use crate::tables::{Feature, StringPool}; + use s2::{cellid::CellID, latlng::LatLng}; + use tempfile::TempDir; + use wkb::writer::{WriteOptions, write_point}; + + let dir = TempDir::new().expect("tempdir"); + let strings = [ + "amenity", + "brand:wikidata", + "car", + "car_repair", + "contact:phone", + "florist", + "Q1", + "Q2", + "shop", + "supermarket", + "055 210 63 10", + "+41 55 210 63 10", + "+41 55 210 63 11", + ]; + let pool = StringPool::create( + strings.iter().map(|s| s.to_string()), + dir.path(), + &dir.path().join("strings"), + ) + .expect("StringPool::create"); + let s2_cell_id = CH_CLOTHES_ATP.s2_cell_id; + let make_feature = |tags: &[(&str, &str)]| -> Feature { + let mut flat_tags = Vec::with_capacity(tags.len() * 2); + for (k, v) in tags { + flat_tags.push(pool.lookup(k).expect("key in pool") as u32); + flat_tags.push(pool.lookup(v).expect("value in pool") as u32); + } + let ll = LatLng::from(CellID(s2_cell_id)); + let mut geometry_wkb = Vec::new(); + write_point( + &mut geometry_wkb, + &geo::Point::new(ll.lng.deg(), ll.lat.deg()), + &WriteOptions { + endianness: wkb::Endianness::LittleEndian, + }, + ) + .expect("wkb encode"); + Feature { + id: 1, + tags: flat_tags, + geometry_wkb, + ..Default::default() + } + }; + let score = |matcher: &PoiMatcher, tags: &[(&str, &str)]| { + let feature = make_feature(tags); + matcher.score_osm_candidate(&OsmCandidate { + feature: &feature, + strings: &pool, + }) + }; + let atp = |tags: &[(&str, &str)]| Place { + s2_cell_id, + spider: String::from("test"), + mask: MatchMask::SHOP, + fetched: test_timestamp(), + shape_wkb: test_shape(s2_cell_id), + tags: self::tags(tags), + }; + + let dealer = atp(&[ + ("addr:country", "CH"), + ("phone", "+41 55 210 63 10"), + ("shop", "car"), + ]); + let m = PoiMatcher::for_place(&dealer, &Nondescript::none()).expect("matcher"); + let same = score( + &m, + &[("shop", "car"), ("contact:phone", "+41 55 210 63 10")], + ); + assert!(same > 0.9 && same < 1.0, "{same}"); + let national = score(&m, &[("shop", "car"), ("contact:phone", "055 210 63 10")]); + assert_eq!(national, same); + let pair = score( + &m, + &[ + ("shop", "car_repair"), + ("contact:phone", "+41 55 210 63 10"), + ], + ); + assert!(pair > 0.9 && pair < same, "{pair}"); + let other_category = score( + &m, + &[ + ("shop", "supermarket"), + ("contact:phone", "+41 55 210 63 10"), + ], + ); + assert_eq!(other_category, 0.0); + let other_phone = score( + &m, + &[("shop", "car"), ("contact:phone", "+41 55 210 63 11")], + ); + assert_eq!(other_phone, 0.0); + // OSM has brand:wikidata, ATP does not: no conflict. + let osm_brand = score( + &m, + &[ + ("brand:wikidata", "Q1"), + ("shop", "car"), + ("contact:phone", "+41 55 210 63 10"), + ], + ); + assert_eq!(osm_brand, same); + + // Both sides have brand:wikidata, and they differ: no match. + let branded = atp(&[ + ("brand:wikidata", "Q2"), + ("phone", "+41 55 210 63 10"), + ("shop", "car"), + ]); + let m = PoiMatcher::for_place(&branded, &Nondescript::none()).expect("matcher"); + let vetoed = score( + &m, + &[ + ("brand:wikidata", "Q1"), + ("shop", "car"), + ("contact:phone", "+41 55 210 63 10"), + ], + ); + assert_eq!(vetoed, 0.0); + let brand = score(&m, &[("brand:wikidata", "Q2"), ("shop", "florist")]); + assert!(brand > same, "{brand}"); + + // No brand:wikidata and no contacts: no matcher at all. + let bare = atp(&[("shop", "florist")]); + assert!(PoiMatcher::for_place(&bare, &Nondescript::none()).is_none()); + } } diff --git a/src/pipeline/atp/mod.rs b/src/pipeline/atp/mod.rs index f3096f40..50494028 100644 --- a/src/pipeline/atp/mod.rs +++ b/src/pipeline/atp/mod.rs @@ -22,7 +22,7 @@ use crate::places::{ParquetWriter, Place}; use crate::{PROGRESS_BAR_STYLE, matchers::MatchMask}; mod fetch; -mod nondescript; +pub(super) mod nondescript; // Only these two re-exported crate-wide (rather than making all of // `fetch` pub(crate)): provenance needs them to assemble this diff --git a/src/pipeline/atp/nondescript/mod.rs b/src/pipeline/atp/nondescript/mod.rs index 2e1717f9..faf599ee 100644 --- a/src/pipeline/atp/nondescript/mod.rs +++ b/src/pipeline/atp/nondescript/mod.rs @@ -39,11 +39,7 @@ use std::sync::LazyLock; use crate::places::{Place, PlaceReader}; use crate::tables::U64Set; -mod emails; -mod phones; - -use emails::{is_email_key, parse_emails}; -use phones::{is_phone_key, parse_phone_numbers}; +use crate::matchers::contact::{is_email_key, is_phone_key, parse_emails, parse_phone_numbers}; /// Places carrying the same value whose bounding box diagonal is at /// least this long make that value nondescript. @@ -59,7 +55,7 @@ pub enum Signal { /// (see [parse_phone_numbers]). Phone, /// Email addresses, encoded as a 64-bit hash of the normalized - /// address (see [parse_emails] and [emails::email_hash]). + /// address (see [parse_emails]). Email, } @@ -281,9 +277,9 @@ pub fn write(nondescript: &Nondescript, workdir: &Path) -> Result<()> { #[cfg(test)] mod tests { - use super::emails::email_hash; use super::*; use crate::matchers::MatchMask; + use crate::matchers::contact::email_hash; use crate::utils::UtcTimestamp; use geo::Coord; diff --git a/src/pipeline/conflate/mod.rs b/src/pipeline/conflate/mod.rs index 7477b6af..ce5c3764 100644 --- a/src/pipeline/conflate/mod.rs +++ b/src/pipeline/conflate/mod.rs @@ -1,7 +1,8 @@ +use super::atp::nondescript::Signal; use super::last_modified; use crate::{ make_progress_bar, - matchers::{OsmCandidate, create_matcher, match_distance}, + matchers::{OsmCandidate, contact::Nondescript, create_matcher, match_distance}, pipeline::EXTERNAL_SORT_CHUNK_BYTES, places::{Place, PlaceReader}, tables::{Feature, OsmFeatures}, @@ -39,12 +40,16 @@ pub fn conflate( workdir: &Path, pipeline_run_id: &str, ) -> Result { - let input_modified = last_modified(&[atp])?.max(osm.modified()?); + let nondescript_phones = Signal::Phone.path(workdir); + let nondescript_emails = Signal::Email.path(workdir); + let input_modified = + last_modified(&[atp, &nondescript_phones, &nondescript_emails])?.max(osm.modified()?); let out_path = workdir.join("conflated.parquet"); if out_path.exists() && last_modified(&[&out_path])? >= input_modified { return Ok(out_path); } + let nondescript = Nondescript::open(&nondescript_phones, &nondescript_emails)?; let atp_reader = PlaceReader::open(atp)?; let num_atp_features = atp_reader.total_rows() as u64; let producer_progress = @@ -56,7 +61,7 @@ pub fn conflate( thread::scope(|s| { let (tx, rx) = sync_channel::(8192); s.spawn(|| { - producer_result = produce_rows(&atp_reader, osm, producer_progress, tx); + producer_result = produce_rows(&atp_reader, osm, &nondescript, producer_progress, tx); }); s.spawn(|| { writer_result = @@ -78,6 +83,7 @@ struct ConflatedFeature { fn produce_rows( atp_reader: &PlaceReader, osm: &OsmFeatures, + nondescript: &Nondescript, progress_bar: ProgressBar, out: SyncSender, ) -> Result<()> { @@ -111,7 +117,7 @@ fn produce_rows( ) }); - let Some(matcher) = create_matcher(atp) else { + let Some(matcher) = create_matcher(atp, nondescript) else { return Ok(()); }; From b129b9aa57fc812d73f59bdb04a34c57bf2a9748 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 25 Oct 2026 13:16:42 +0000 Subject: [PATCH 2/3] fix(deps): allow atomic-polyfill in cargo deny advisories Ignore RUSTSEC-2023-0089 (atomic-polyfill is unmaintained), which phonenumber pulls in through postcard and heapless 0.7, only for targets without native atomics. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01P43kACSyJVwszya8Z3d4qQ --- deny.toml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/deny.toml b/deny.toml index bece6f4b..9c5d59c5 100644 --- a/deny.toml +++ b/deny.toml @@ -28,6 +28,13 @@ ignore = [ # release picks it up, then drop these two lines. "RUSTSEC-2026-0194", # quadratic run time checking duplicate attribute names "RUSTSEC-2026-0195", # unbounded namespace-declaration allocation (OOM) + # atomic-polyfill is unmaintained, not vulnerable. It comes in via + # phonenumber -> postcard -> heapless 0.7, which only uses it on targets + # without native atomics, so it is never built for the x86_64 and aarch64 + # targets we ship (`cargo tree -i atomic-polyfill` prints nothing there; + # only `--target all` shows it). Drop this line once postcard moves to + # heapless 0.8, or phonenumber stops using postcard. + "RUSTSEC-2023-0089", ] # ── licenses: only accept licenses we've reviewed as OK to redistribute ───── From 0b04ded0799699bcf2f7419a953e08f14934dc7e Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 25 Oct 2026 13:14:09 +0000 Subject: [PATCH 3/3] fix(matchers): parse single OSM phone numbers without an iterator parse_osm_phone now calls phones::parse_phone_number directly instead of taking the first item of parse_phone_numbers. With build-mode none, CodeQL resolved that `.next()` to unrelated iterators in tables::graph and reported a cleartext-logging flow that does not exist. Also fill in the planet match count of this change in the note. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01P43kACSyJVwszya8Z3d4qQ --- docs/notes/2026-10-25-phone-and-email-matching.md | 6 ++++-- src/matchers/contact/mod.rs | 8 ++++---- src/matchers/contact/phones.rs | 4 +++- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/docs/notes/2026-10-25-phone-and-email-matching.md b/docs/notes/2026-10-25-phone-and-email-matching.md index c5be75e8..0574c323 100644 --- a/docs/notes/2026-10-25-phone-and-email-matching.md +++ b/docs/notes/2026-10-25-phone-and-email-matching.md @@ -94,9 +94,11 @@ ATP run `2026-10-10-13-32-41` against the planet of the same week, |---|---:| | `brand:wikidata`, shops only (before) | 825,901 | | `brand:wikidata`, wider categories | 1,233,452 | -| `brand:wikidata` or phone and email, wider categories (this change) | PLANET_PR | +| `brand:wikidata` or phone and email, wider categories (this change) | 1,302,469 | -No match from before was lost. To measure precision without hand +So phone and email add 69,017 matches on top of the wider categories. +No match from before was lost, and 535 moved to another OSM feature. +The match step took 38 s for 5.3 million ATP features. To measure precision without hand labels, a test run hid the brand tags on both sides and checked whether contact evidence alone finds the OSM feature that `brand:wikidata` found. Of about 183,000 such rows, it found the same diff --git a/src/matchers/contact/mod.rs b/src/matchers/contact/mod.rs index 9e054812..51aa2318 100644 --- a/src/matchers/contact/mod.rs +++ b/src/matchers/contact/mod.rs @@ -24,7 +24,7 @@ mod phones; #[cfg(test)] pub(crate) use emails::email_hash; pub(crate) use emails::{is_email_key, parse_emails}; -pub(crate) use phones::{is_phone_key, parse_phone_numbers}; +pub(crate) use phones::{is_phone_key, parse_phone_number, parse_phone_numbers}; /// Score factor of a contact match whose main categories are equal. /// Below 1.0, so a `brand:wikidata` match at the same distance wins. @@ -135,7 +135,7 @@ impl AtpContacts { } /// Parses one OSM phone value. Most are in international format, which -/// [parse_phone_numbers] reads quickly. The rest, such as `044 417 77 +/// [parse_phone_number] reads quickly. The rest, such as `044 417 77 /// 77`, go through the `phonenumber` crate in the AllThePlaces /// feature's country. That is slow (about 60 µs a call) but rare: on /// the planet, it adds about one second in all. In some countries, @@ -144,14 +144,14 @@ impl AtpContacts { fn parse_osm_phone(part: &str, country: Option) -> Option { let part = part.trim(); if part.starts_with('+') { - return parse_phone_numbers(part).next(); + return parse_phone_number(part); } let number = phonenumber::parse(country, part).ok()?; if !phonenumber::is_valid(&number) { return None; } let e164 = number.format().mode(phonenumber::Mode::E164).to_string(); - parse_phone_numbers(&e164).next() + parse_phone_number(&e164) } /// Keys whose value is a feature's main category, in order of priority. diff --git a/src/matchers/contact/phones.rs b/src/matchers/contact/phones.rs index ec6c8f72..d95953db 100644 --- a/src/matchers/contact/phones.rs +++ b/src/matchers/contact/phones.rs @@ -36,7 +36,9 @@ pub(crate) fn parse_phone_numbers(value: &str) -> impl Iterator + '_ value.split(';').filter_map(parse_phone_number) } -fn parse_phone_number(part: &str) -> Option { +/// Parses one phone number in international format, as in +/// [parse_phone_numbers]. +pub(crate) fn parse_phone_number(part: &str) -> Option { let rest = part.trim().strip_prefix('+')?; let mut number: u64 = 0; let mut digits = 0;