From 8d6a1ce340f182f82db0359de7264674f72e742e Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 11 Nov 2026 12:48:25 +0000 Subject: [PATCH 1/2] feat(wikidata): Count OSM features and names per brand collect_wikidata_ids already visits every ATP and OSM feature with a brand:wikidata, operator:wikidata or network:wikidata tag. It now also writes two files for the spider stats: - osm-brands.tsv: OSM features per Wikidata ID, country and main category (such as shop=supermarket). A feature that names the same ID twice counts once. - brand-names.tsv: the three most common brand, operator or network names per Wikidata ID, country and key, with how often OSM and ATP use each. Names count only where the Wikidata tag has a single ID. The country of an OSM feature comes from its centroid, so features with a Wikidata tag now get their geometry decoded. Part of #924. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Ttfb2LhDRExioXkdHbxLno --- docs/TECHNICAL_DESIGN.md | 10 ++ src/pipeline/brand_totals.rs | 287 +++++++++++++++++++++++++++++++++++ src/pipeline/mod.rs | 1 + src/pipeline/wikidata_ids.rs | 165 ++++++++++++++------ 4 files changed, 420 insertions(+), 43 deletions(-) create mode 100644 src/pipeline/brand_totals.rs diff --git a/docs/TECHNICAL_DESIGN.md b/docs/TECHNICAL_DESIGN.md index 0c03cd3..df50a53 100644 --- a/docs/TECHNICAL_DESIGN.md +++ b/docs/TECHNICAL_DESIGN.md @@ -211,6 +211,8 @@ graph TD ATP_PARQUET --> COLLECT_WIKI(collect_wikidata_ids) OSM_INDEX --> COLLECT_WIKI COLLECT_WIKI --> WIKIDATA[wikidata-ids] + COLLECT_WIKI --> OSM_BRANDS[osm-brands.tsv] + COLLECT_WIKI --> BRAND_NAMES[brand-names.tsv] WIKIDATA --> QRANK_QUANTILES(compute_qrank_quantiles) QRANK_CSV --> QRANK_QUANTILES QRANK_QUANTILES --> QRANK_Q[qrank-quantiles] @@ -395,6 +397,14 @@ fetch away. ([#713](https://github.com/brawer/osmdiffs/issues/713)). Plain `wikidata` is left out, since on OSM it mostly points at cities, rivers and the like. Visits every feature in `osm-features.index`. + On the side, it writes `osm-brands.tsv`, the number of OSM features + per Wikidata ID, country and main category (such as + `shop=supermarket`), and `brand-names.tsv`, the three most common + `brand`, `operator` or `network` names per ID, country and key, with + how often OSM and ATP use them + ([`brand_totals.rs`](../src/pipeline/brand_totals.rs)). The spider + stats compare ATP with the first, and show the second so that + readers needn't look up Wikidata IDs. - **`compute_qrank_quantiles`** ([`src/pipeline/qrank/mod.rs`](../src/pipeline/qrank/mod.rs)) — gives each ID in `wikidata-ids` that has pageviews its QRank quantile among diff --git a/src/pipeline/brand_totals.rs b/src/pipeline/brand_totals.rs new file mode 100644 index 0000000..0a3580a --- /dev/null +++ b/src/pipeline/brand_totals.rs @@ -0,0 +1,287 @@ +//! For each brand, operator and network that features refer to by +//! Wikidata ID: how many OpenStreetMap features it has per country and +//! main category, and the names that OSM and AllThePlaces give it. The +//! spider stats compare these totals with AllThePlaces, and show the +//! names so that their readers needn't look up Wikidata IDs. +//! +//! [super::wikidata_ids::collect_wikidata_ids] fills them in, since it +//! visits the same features anyway. + +use crate::matchers::{contact::main_category, parse_wikidata_ids}; +use crate::utils::countries::Country; +use anyhow::{Context, Result}; +use std::collections::HashMap; +use std::io::{BufWriter, Write}; +use std::path::{Path, PathBuf}; + +use super::wikidata_ids::WIKIDATA_KEYS; + +/// Filename, within `workdir`, of the OSM feature totals. +const FEATURES_FILENAME: &str = "osm-brands.tsv"; + +/// Filename, within `workdir`, of the names. +const NAMES_FILENAME: &str = "brand-names.tsv"; + +/// The most names kept per Wikidata ID, country and key. +const MAX_NAMES: usize = 3; + +pub(super) fn features_path(workdir: &Path) -> PathBuf { + workdir.join(FEATURES_FILENAME) +} + +pub(super) fn names_path(workdir: &Path) -> PathBuf { + workdir.join(NAMES_FILENAME) +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum Source { + Osm, + Atp, +} + +/// Wikidata ID, country, and the key of the name, such as `brand`. +type NameKey = (u64, Option, &'static str); + +#[derive(Debug, Default)] +pub(super) struct BrandTotals { + /// OSM features per Wikidata ID, country and main category (see + /// [main_category]). A feature that refers to the same ID twice, + /// such as by `brand:wikidata` and `operator:wikidata`, counts once. + features: HashMap<(u64, Option, Option), u64>, + + /// How often OSM and ATP features use each name, in that order. + names: HashMap>, +} + +impl BrandTotals { + /// Adds a feature with `tags` that lies in `country`. Names count + /// only where the Wikidata tag names a single ID: with several, + /// there's no telling which name goes with which. + pub fn add<'a>( + &mut self, + source: Source, + tags: impl Iterator + Clone, + country: Option, + ) { + let mut ids: Vec = Vec::new(); + for key in WIKIDATA_KEYS { + let Some((_, value)) = tags.clone().find(|(k, _)| *k == key) else { + continue; + }; + let found: Vec = parse_wikidata_ids(value).collect(); + let name_key = key.strip_suffix(":wikidata").expect("Wikidata key"); + if let [id] = found[..] + && let Some((_, name)) = tags.clone().find(|(k, _)| *k == name_key) + { + let name = clean_name(name); + if !name.is_empty() { + let counts = self + .names + .entry((id, country, name_key)) + .or_default() + .entry(name) + .or_default(); + counts[source as usize] += 1; + } + } + ids.extend(found); + } + if source != Source::Osm || ids.is_empty() { + return; + } + ids.sort_unstable(); + ids.dedup(); + let category = main_category(tags); + for id in ids { + *self + .features + .entry((id, country, category.clone())) + .or_default() += 1; + } + } + + pub fn merge(mut self, other: BrandTotals) -> BrandTotals { + if self.features.len() + self.names.len() < other.features.len() + other.names.len() { + return other.merge(self); + } + for (key, n) in other.features { + *self.features.entry(key).or_default() += n; + } + for (key, names) in other.names { + let mine = self.names.entry(key).or_default(); + for (name, [osm, atp]) in names { + let counts = mine.entry(name).or_default(); + counts[0] += osm; + counts[1] += atp; + } + } + self + } + + /// Writes the feature totals to [features_path] and the most + /// common names to [names_path], both sorted, and returns how many + /// lines each has. + pub fn write(&self, workdir: &Path) -> Result<(usize, usize)> { + let mut features: Vec<_> = self.features.iter().collect(); + features.sort_by_cached_key(|((id, country, category), _)| { + (*id, code(*country), category.clone()) + }); + let num_features = features.len(); + write_tsv( + &features_path(workdir), + "wikidata_id\tcountry\tcategory\tfeatures", + features.into_iter().map(|((id, country, category), n)| { + format!( + "Q{id}\t{}\t{}\t{n}", + code(*country), + category.as_deref().unwrap_or("") + ) + }), + )?; + + let mut keys: Vec<&NameKey> = self.names.keys().collect(); + keys.sort_by_cached_key(|(id, country, key)| (*id, code(*country), *key)); + let mut lines = Vec::new(); + for k in keys { + let mut names: Vec<(&String, &[u64; 2])> = self.names[k].iter().collect(); + names.sort_by(|a, b| (b.1[0] + b.1[1]).cmp(&(a.1[0] + a.1[1])).then(a.0.cmp(b.0))); + for (name, [osm, atp]) in names.into_iter().take(MAX_NAMES) { + let (id, country, key) = k; + lines.push(format!( + "Q{id}\t{}\t{key}\t{name}\t{osm}\t{atp}", + code(*country) + )); + } + } + let num_names = lines.len(); + write_tsv( + &names_path(workdir), + "wikidata_id\tcountry\tkey\tname\tosm\tatp", + lines.into_iter(), + )?; + Ok((num_features, num_names)) + } +} + +fn code(country: Option) -> String { + country.map(|c| c.code().to_string()).unwrap_or_default() +} + +/// `name` without surrounding spaces, and with tabs and line breaks +/// replaced, so it fits into one TSV column. +fn clean_name(name: &str) -> String { + name.trim() + .chars() + .map(|c| if c.is_control() { ' ' } else { c }) + .collect() +} + +fn write_tsv(out: &Path, header: &str, lines: impl Iterator) -> Result<()> { + let mut tmp = out.to_path_buf(); + tmp.add_extension("tmp"); + let mut w = BufWriter::new( + std::fs::File::create(&tmp).with_context(|| format!("could not create {tmp:?}"))?, + ); + writeln!(w, "{header}")?; + for line in lines { + writeln!(w, "{line}")?; + } + w.into_inner()?.sync_all()?; + std::fs::rename(&tmp, out)?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + const CH: Option = Some(Country::SWITZERLAND); + const DE: Option = Some(Country::GERMANY); + + #[test] + fn test_brand_totals() -> Result<()> { + let mut a = BrandTotals::default(); + let mut b = BrandTotals::default(); + let migros = [ + ("shop", "supermarket"), + ("brand", "Migros"), + ("brand:wikidata", "Q680727"), + ("operator", "Genossenschaft Migros Zürich"), + ("operator:wikidata", "Q1"), + ]; + a.add(Source::Osm, migros.iter().copied(), CH); + b.add(Source::Osm, migros.iter().copied(), CH); + let typo = [ + ("shop", "supermarket"), + ("brand", "Migors"), + ("brand:wikidata", "Q680727"), + ]; + b.add(Source::Osm, typo.iter().copied(), CH); + // ATP names count, but ATP features don't. + let atp = [ + ("shop", "supermarket"), + ("brand", "Migros\t"), + ("brand:wikidata", "Q680727"), + ]; + b.add(Source::Atp, atp.iter().copied(), CH); + // Two IDs: no name. + let two = [ + ("amenity", "fuel"), + ("brand", "Migrol;Migros"), + ("brand:wikidata", "Q1;Q680727"), + ]; + b.add(Source::Osm, two.iter().copied(), DE); + // The same ID twice counts once. + let same = [ + ("brand", "Migros"), + ("brand:wikidata", "Q680727"), + ("operator:wikidata", "Q680727"), + ]; + b.add(Source::Osm, same.iter().copied(), None); + let totals = a.merge(b); + + let dir = tempfile::tempdir()?; + assert_eq!(totals.write(dir.path())?, (5, 4)); + assert_eq!( + std::fs::read_to_string(features_path(dir.path()))?, + "wikidata_id\tcountry\tcategory\tfeatures\n\ + Q1\tCH\tshop=supermarket\t2\n\ + Q1\tDE\tamenity=fuel\t1\n\ + Q680727\t\t\t1\n\ + Q680727\tCH\tshop=supermarket\t3\n\ + Q680727\tDE\tamenity=fuel\t1\n" + ); + assert_eq!( + std::fs::read_to_string(names_path(dir.path()))?, + "wikidata_id\tcountry\tkey\tname\tosm\tatp\n\ + Q1\tCH\toperator\tGenossenschaft Migros Zürich\t2\t0\n\ + Q680727\t\tbrand\tMigros\t1\t0\n\ + Q680727\tCH\tbrand\tMigros\t2\t1\n\ + Q680727\tCH\tbrand\tMigors\t1\t0\n" + ); + Ok(()) + } + + #[test] + fn test_max_names() -> Result<()> { + let mut totals = BrandTotals::default(); + for (name, n) in [("A", 1), ("B", 4), ("C", 2), ("D", 3)] { + for _ in 0..n { + totals.add( + Source::Osm, + [("brand", name), ("brand:wikidata", "Q7")].into_iter(), + CH, + ); + } + } + let dir = tempfile::tempdir()?; + totals.write(dir.path())?; + let names: Vec = std::fs::read_to_string(names_path(dir.path()))? + .lines() + .skip(1) + .map(|l| l.split('\t').nth(3).unwrap().to_string()) + .collect(); + assert_eq!(names, vec!["B", "D", "C"]); + Ok(()) + } +} diff --git a/src/pipeline/mod.rs b/src/pipeline/mod.rs index 51aee30..35eec22 100644 --- a/src/pipeline/mod.rs +++ b/src/pipeline/mod.rs @@ -31,6 +31,7 @@ use time::UtcDateTime; pub(crate) const EXTERNAL_SORT_CHUNK_BYTES: usize = 512 * 1024 * 1024; mod atp; +mod brand_totals; mod conflate; mod conflated_tiles; mod datapackage; diff --git a/src/pipeline/wikidata_ids.rs b/src/pipeline/wikidata_ids.rs index c17e38d..85bf53b 100644 --- a/src/pipeline/wikidata_ids.rs +++ b/src/pipeline/wikidata_ids.rs @@ -17,12 +17,19 @@ //! point at cities, rivers, churches and the like rather than at //! brands. So are other `*:wikidata` keys such as `species:wikidata` or //! `trough:material:wikidata`, which are about something else entirely. +//! +//! On the side, it counts the OSM features of each ID per country and +//! main category, and the names that features give it; see +//! [super::brand_totals]. +use super::brand_totals::{self, BrandTotals, Source}; use crate::make_progress_bar; -use crate::matchers::parse_wikidata_ids; -use crate::places::PlaceReader; +use crate::matchers::{OsmCandidate, parse_wikidata_ids}; +use crate::places::{Place, PlaceReader}; use crate::tables::{OsmFeatures, U64Set}; +use crate::utils::countries::{Country, country}; use anyhow::Result; +use geo::Centroid; use indicatif::MultiProgress; use rayon::prelude::*; use std::path::{Path, PathBuf}; @@ -43,7 +50,8 @@ pub(super) fn is_wikidata_key(key: &str) -> bool { /// Collects brand/operator/network Wikidata IDs from both `atp` /// (`alltheplaces.parquet`) and `osm`, and writes their union to -/// `workdir/wikidata-ids` as a [U64Set]. +/// `workdir/wikidata-ids` as a [U64Set], and their [BrandTotals] next +/// to it. pub fn collect_wikidata_ids( atp: &Path, osm: &OsmFeatures, @@ -52,14 +60,24 @@ pub fn collect_wikidata_ids( ) -> Result { let out = workdir.join(WIKIDATA_IDS_FILENAME); let input_modified = super::last_modified(&[atp])?.max(osm.modified()?); - if out.exists() && super::last_modified(&[&out])? >= input_modified { + let outputs = [ + out.clone(), + brand_totals::features_path(workdir), + brand_totals::names_path(workdir), + ]; + if outputs.iter().all(|p| p.exists()) + && super::last_modified(&outputs.iter().map(PathBuf::as_path).collect::>())? + >= input_modified + { return Ok(out); } - let mut ids = collect_from_atp(atp)?; + let (mut ids, atp_totals) = collect_from_atp(atp)?; let num_atp_ids = ids.len(); - ids.extend(collect_from_osm(osm, progress)?); + let (osm_ids, osm_totals) = collect_from_osm(osm, progress)?; + ids.extend(osm_ids); let num_osm_ids = ids.len() - num_atp_ids; + let (brand_features, brand_names) = osm_totals.merge(atp_totals).write(workdir)?; // Sole external sort in this step (see `run_step("collect_wikidata_ids", // ...)` in pipeline/mod.rs), so it gets the full chunk-size budget. @@ -73,41 +91,73 @@ pub fn collect_wikidata_ids( log::info!( atp_tags = num_atp_ids, osm_tags = num_osm_ids, - unique_ids = unique; + unique_ids = unique, + brand_features_lines = brand_features, + brand_names_lines = brand_names; "collect_wikidata_ids: done" ); Ok(out) } /// The Wikidata IDs in `atp`'s brand/operator/network tags, with -/// duplicates. -fn collect_from_atp(atp: &Path) -> Result> { +/// duplicates, and the names that go with them. +fn collect_from_atp(atp: &Path) -> Result<(Vec, BrandTotals)> { let reader = PlaceReader::open(atp)?; let mut ids = Vec::new(); + let mut totals = BrandTotals::default(); for batch in reader.read_all()? { - for place in batch? { - for (key, value) in &place.tags { - if is_wikidata_key(key) { - ids.extend(parse_wikidata_ids(value)); - } - } - } + let (batch_ids, batch_totals) = batch? + .par_iter() + .fold( + || (Vec::new(), BrandTotals::default()), + |(mut ids, mut totals), place| { + let len = ids.len(); + for (key, value) in &place.tags { + if is_wikidata_key(key) { + ids.extend(parse_wikidata_ids(value)); + } + } + if ids.len() > len { + let tags = place.tags.iter().map(|(k, v)| (k.as_str(), v.as_str())); + totals.add(Source::Atp, tags, place_country(place)); + } + (ids, totals) + }, + ) + .reduce( + || (Vec::new(), BrandTotals::default()), + |(mut a, ta), (b, tb)| { + a.extend(b); + (a, ta.merge(tb)) + }, + ); + ids.extend(batch_ids); + totals = totals.merge(batch_totals); } - Ok(ids) + Ok((ids, totals)) +} + +fn place_country(place: &Place) -> Option { + let ll = s2::latlng::LatLng::from(s2::cellid::CellID(place.s2_cell_id)); + country(ll.lng.deg(), ll.lat.deg()) } /// The Wikidata IDs in `osm`'s brand/operator/network tags, with -/// duplicates. Visits every feature in the index, in parallel; tag -/// keys are compared as string pool indices, so only the values of -/// matching tags get looked up as strings. -fn collect_from_osm(osm: &OsmFeatures, progress: &MultiProgress) -> Result> { +/// duplicates, and their [BrandTotals]. Visits every feature in the +/// index, in parallel; tag keys are compared as string pool indices, so +/// only features with matching tags get their tags looked up as strings +/// and their geometry decoded, for the country. +fn collect_from_osm( + osm: &OsmFeatures, + progress: &MultiProgress, +) -> Result<(Vec, BrandTotals)> { let key_ids: Vec = WIKIDATA_KEYS .iter() .filter_map(|key| osm.strings.lookup(key)) .map(|id| id as u32) .collect(); if key_ids.is_empty() { - return Ok(Vec::new()); + return Ok((Vec::new(), BrandTotals::default())); } let progress_bar = make_progress_bar( @@ -116,28 +166,48 @@ fn collect_from_osm(osm: &OsmFeatures, progress: &MultiProgress) -> Result = feature - .tags - .as_chunks::<2>() - .0 - .iter() - .filter(|[k, _]| key_ids.contains(k)) - .flat_map(|[_, v]| parse_wikidata_ids(osm.strings.get(*v as usize))) - .collect(); - Ok::<_, anyhow::Error>(ids) - }) - .try_reduce(Vec::new, |mut a, b| { - a.extend(b); - Ok(a) - })?; + .try_fold( + || (Vec::new(), BrandTotals::default()), + |(mut ids, mut totals), r| { + progress_bar.inc(1); + let feature = osm.index.get_feature(r)?; + let len = ids.len(); + ids.extend( + feature + .tags + .as_chunks::<2>() + .0 + .iter() + .filter(|[k, _]| key_ids.contains(k)) + .flat_map(|[_, v]| parse_wikidata_ids(osm.strings.get(*v as usize))), + ); + if ids.len() > len { + let candidate = OsmCandidate { + feature: &feature, + strings: &osm.strings, + }; + let country = candidate + .geometry() + .ok() + .and_then(|g| g.centroid()) + .and_then(|c| country(c.x(), c.y())); + totals.add(Source::Osm, candidate.tags(), country); + } + Ok::<_, anyhow::Error>((ids, totals)) + }, + ) + .try_reduce( + || (Vec::new(), BrandTotals::default()), + |(mut a, ta), (b, tb)| { + a.extend(b); + Ok((a, ta.merge(tb))) + }, + )?; progress_bar.finish(); - Ok(ids) + Ok(result) } #[cfg(test)] @@ -222,7 +292,7 @@ mod tests { &[("shop", "supermarket"), ("brand:wikidata", "Q1")], ], )?; - let mut ids = collect_from_osm(&osm, &MultiProgress::new())?; + let (mut ids, _) = collect_from_osm(&osm, &MultiProgress::new())?; ids.sort(); assert_eq!(ids, vec![1, 1, 2, 3, 4]); Ok(()) @@ -232,7 +302,7 @@ mod tests { fn test_collect_from_osm_without_wikidata_keys() -> Result<()> { let dir = TempDir::new()?; let osm = build_osm(dir.path(), &[&[("shop", "bakery")]])?; - assert!(collect_from_osm(&osm, &MultiProgress::new())?.is_empty()); + assert!(collect_from_osm(&osm, &MultiProgress::new())?.0.is_empty()); Ok(()) } @@ -275,6 +345,15 @@ mod tests { assert!(set.contains(680727)); assert!(!set.contains(72)); + let features = std::fs::read_to_string(brand_totals::features_path(workdir.path()))?; + assert_eq!(features.lines().count(), 1 + 2, "{features}"); + assert!(features.contains("Q680727\t\tshop=supermarket\t1\n")); + let names = std::fs::read_to_string(brand_totals::names_path(workdir.path()))?; + assert!( + names.contains("Q379911\tCH\tbrand\tDenner\t0\t1\n"), + "{names}" + ); + // Calling again should reuse the existing file, not fail or // recompute. let modified = std::fs::metadata(&out)?.modified()?; From cd3dc56375cc9b4104010e12fc32ad32bfda12ee Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 11 Nov 2026 12:54:44 +0000 Subject: [PATCH 2/2] refactor(wikidata): Keep one display name per brand The names are only for telling brands apart in the spider stats, so brand-names.tsv now has one name per Wikidata ID and key, not three per country: the most common name in Latin script, or else the most common one. ATP places no longer need their country looked up. osm-brands.tsv keeps its countries, so the spider stats can compare a per-country spider with OSM in that country. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Ttfb2LhDRExioXkdHbxLno --- docs/TECHNICAL_DESIGN.md | 6 +- src/pipeline/brand_totals.rs | 126 ++++++++++++++++++++--------------- src/pipeline/wikidata_ids.rs | 19 ++---- 3 files changed, 82 insertions(+), 69 deletions(-) diff --git a/docs/TECHNICAL_DESIGN.md b/docs/TECHNICAL_DESIGN.md index df50a53..5eced07 100644 --- a/docs/TECHNICAL_DESIGN.md +++ b/docs/TECHNICAL_DESIGN.md @@ -399,9 +399,9 @@ fetch away. rivers and the like. Visits every feature in `osm-features.index`. On the side, it writes `osm-brands.tsv`, the number of OSM features per Wikidata ID, country and main category (such as - `shop=supermarket`), and `brand-names.tsv`, the three most common - `brand`, `operator` or `network` names per ID, country and key, with - how often OSM and ATP use them + `shop=supermarket`), and `brand-names.tsv`, one `brand`, + `operator` or `network` name per ID and key: the most common one in + Latin script, or else the most common one, from OSM and ATP ([`brand_totals.rs`](../src/pipeline/brand_totals.rs)). The spider stats compare ATP with the first, and show the second so that readers needn't look up Wikidata IDs. diff --git a/src/pipeline/brand_totals.rs b/src/pipeline/brand_totals.rs index 0a3580a..dc650bd 100644 --- a/src/pipeline/brand_totals.rs +++ b/src/pipeline/brand_totals.rs @@ -1,6 +1,6 @@ //! For each brand, operator and network that features refer to by //! Wikidata ID: how many OpenStreetMap features it has per country and -//! main category, and the names that OSM and AllThePlaces give it. The +//! main category, and a name for it, from OSM or AllThePlaces. The //! spider stats compare these totals with AllThePlaces, and show the //! names so that their readers needn't look up Wikidata IDs. //! @@ -22,9 +22,6 @@ const FEATURES_FILENAME: &str = "osm-brands.tsv"; /// Filename, within `workdir`, of the names. const NAMES_FILENAME: &str = "brand-names.tsv"; -/// The most names kept per Wikidata ID, country and key. -const MAX_NAMES: usize = 3; - pub(super) fn features_path(workdir: &Path) -> PathBuf { workdir.join(FEATURES_FILENAME) } @@ -39,8 +36,8 @@ pub(super) enum Source { Atp, } -/// Wikidata ID, country, and the key of the name, such as `brand`. -type NameKey = (u64, Option, &'static str); +/// Wikidata ID, and the key of the name, such as `brand`. +type NameKey = (u64, &'static str); #[derive(Debug, Default)] pub(super) struct BrandTotals { @@ -54,9 +51,10 @@ pub(super) struct BrandTotals { } impl BrandTotals { - /// Adds a feature with `tags` that lies in `country`. Names count - /// only where the Wikidata tag names a single ID: with several, - /// there's no telling which name goes with which. + /// Adds a feature with `tags` that lies in `country`, which only + /// matters for OSM features. Names count only where the Wikidata + /// tag names a single ID: with several, there's no telling which + /// name goes with which. pub fn add<'a>( &mut self, source: Source, @@ -77,7 +75,7 @@ impl BrandTotals { if !name.is_empty() { let counts = self .names - .entry((id, country, name_key)) + .entry((id, name_key)) .or_default() .entry(name) .or_default(); @@ -118,9 +116,11 @@ impl BrandTotals { self } - /// Writes the feature totals to [features_path] and the most - /// common names to [names_path], both sorted, and returns how many - /// lines each has. + /// Writes the feature totals to [features_path] and a name per + /// Wikidata ID and key to [names_path], both sorted, and returns + /// how many lines each has. The name is the most common one in + /// Latin script, or if there is none, the most common one: enough + /// to tell brands apart for readers of the spider stats. pub fn write(&self, workdir: &Path) -> Result<(usize, usize)> { let mut features: Vec<_> = self.features.iter().collect(); features.sort_by_cached_key(|((id, country, category), _)| { @@ -140,29 +140,44 @@ impl BrandTotals { )?; let mut keys: Vec<&NameKey> = self.names.keys().collect(); - keys.sort_by_cached_key(|(id, country, key)| (*id, code(*country), *key)); - let mut lines = Vec::new(); - for k in keys { - let mut names: Vec<(&String, &[u64; 2])> = self.names[k].iter().collect(); - names.sort_by(|a, b| (b.1[0] + b.1[1]).cmp(&(a.1[0] + a.1[1])).then(a.0.cmp(b.0))); - for (name, [osm, atp]) in names.into_iter().take(MAX_NAMES) { - let (id, country, key) = k; - lines.push(format!( - "Q{id}\t{}\t{key}\t{name}\t{osm}\t{atp}", - code(*country) - )); - } - } + keys.sort(); + let lines: Vec = keys + .into_iter() + .filter_map(|k| { + let (name, [osm, atp]) = display_name(&self.names[k])?; + let (id, key) = k; + Some(format!("Q{id}\t{key}\t{name}\t{osm}\t{atp}")) + }) + .collect(); let num_names = lines.len(); write_tsv( &names_path(workdir), - "wikidata_id\tcountry\tkey\tname\tosm\tatp", + "wikidata_id\tkey\tname\tosm\tatp", lines.into_iter(), )?; Ok((num_features, num_names)) } } +/// The name to show, out of `names` with their counts: the most +/// common in Latin script, or else the most common, with ties broken +/// by the name itself. +fn display_name(names: &HashMap) -> Option<(&String, [u64; 2])> { + let rank = |(name, counts): &(&String, &[u64; 2])| (is_latin(name), counts[0] + counts[1]); + names + .iter() + .max_by(|a, b| rank(a).cmp(&rank(b)).then(b.0.cmp(a.0))) + .map(|(name, counts)| (name, *counts)) +} + +/// Whether every letter of `name` is in Latin script, such as +/// `Migros` or `Żabka`, unlike `ミニストップ`. +fn is_latin(name: &str) -> bool { + name.chars().filter(|c| c.is_alphabetic()).all( + |c| matches!(c, 'A'..='Z' | 'a'..='z' | '\u{00C0}'..='\u{024F}' | '\u{1E00}'..='\u{1EFF}'), + ) +} + fn code(country: Option) -> String { country.map(|c| c.code().to_string()).unwrap_or_default() } @@ -241,7 +256,7 @@ mod tests { let totals = a.merge(b); let dir = tempfile::tempdir()?; - assert_eq!(totals.write(dir.path())?, (5, 4)); + assert_eq!(totals.write(dir.path())?, (5, 2)); assert_eq!( std::fs::read_to_string(features_path(dir.path()))?, "wikidata_id\tcountry\tcategory\tfeatures\n\ @@ -253,35 +268,40 @@ mod tests { ); assert_eq!( std::fs::read_to_string(names_path(dir.path()))?, - "wikidata_id\tcountry\tkey\tname\tosm\tatp\n\ - Q1\tCH\toperator\tGenossenschaft Migros Zürich\t2\t0\n\ - Q680727\t\tbrand\tMigros\t1\t0\n\ - Q680727\tCH\tbrand\tMigros\t2\t1\n\ - Q680727\tCH\tbrand\tMigors\t1\t0\n" + "wikidata_id\tkey\tname\tosm\tatp\n\ + Q1\toperator\tGenossenschaft Migros Zürich\t2\t0\n\ + Q680727\tbrand\tMigros\t3\t1\n" ); Ok(()) } #[test] - fn test_max_names() -> Result<()> { - let mut totals = BrandTotals::default(); - for (name, n) in [("A", 1), ("B", 4), ("C", 2), ("D", 3)] { - for _ in 0..n { - totals.add( - Source::Osm, - [("brand", name), ("brand:wikidata", "Q7")].into_iter(), - CH, - ); - } - } - let dir = tempfile::tempdir()?; - totals.write(dir.path())?; - let names: Vec = std::fs::read_to_string(names_path(dir.path()))? - .lines() - .skip(1) - .map(|l| l.split('\t').nth(3).unwrap().to_string()) - .collect(); - assert_eq!(names, vec!["B", "D", "C"]); - Ok(()) + fn test_display_name() { + let shown = |list: &[(&str, u64)]| { + let names: HashMap = + list.iter().map(|(n, c)| (n.to_string(), [*c, 0])).collect(); + display_name(&names).map(|(n, _)| n.clone()) + }; + assert_eq!(shown(&[("A", 1), ("B", 4), ("C", 4)]), Some("B".into())); + assert_eq!( + shown(&[("ミニストップ", 9), ("Ministop", 2)]), + Some("Ministop".into()) + ); + assert_eq!( + shown(&[("ミニストップ", 9), ("ローソン", 2)]), + Some("ミニストップ".into()) + ); + assert_eq!(shown(&[]), None); + } + + #[test] + fn test_is_latin() { + assert!(is_latin("Migros")); + assert!(is_latin("Coop Pronto 24/7")); + assert!(is_latin("Żabka")); + assert!(is_latin("Phở Hòa")); + assert!(!is_latin("ミニストップ")); + assert!(!is_latin("Пятёрочка")); + assert!(!is_latin("7-Eleven セブン")); } } diff --git a/src/pipeline/wikidata_ids.rs b/src/pipeline/wikidata_ids.rs index 85bf53b..d7f1633 100644 --- a/src/pipeline/wikidata_ids.rs +++ b/src/pipeline/wikidata_ids.rs @@ -25,9 +25,9 @@ use super::brand_totals::{self, BrandTotals, Source}; use crate::make_progress_bar; use crate::matchers::{OsmCandidate, parse_wikidata_ids}; -use crate::places::{Place, PlaceReader}; +use crate::places::PlaceReader; use crate::tables::{OsmFeatures, U64Set}; -use crate::utils::countries::{Country, country}; +use crate::utils::countries::country; use anyhow::Result; use geo::Centroid; use indicatif::MultiProgress; @@ -100,7 +100,8 @@ pub fn collect_wikidata_ids( } /// The Wikidata IDs in `atp`'s brand/operator/network tags, with -/// duplicates, and the names that go with them. +/// duplicates, and the names that go with them. Unlike for OSM, the +/// country doesn't matter here, so it isn't looked up. fn collect_from_atp(atp: &Path) -> Result<(Vec, BrandTotals)> { let reader = PlaceReader::open(atp)?; let mut ids = Vec::new(); @@ -119,7 +120,7 @@ fn collect_from_atp(atp: &Path) -> Result<(Vec, BrandTotals)> { } if ids.len() > len { let tags = place.tags.iter().map(|(k, v)| (k.as_str(), v.as_str())); - totals.add(Source::Atp, tags, place_country(place)); + totals.add(Source::Atp, tags, None); } (ids, totals) }, @@ -137,11 +138,6 @@ fn collect_from_atp(atp: &Path) -> Result<(Vec, BrandTotals)> { Ok((ids, totals)) } -fn place_country(place: &Place) -> Option { - let ll = s2::latlng::LatLng::from(s2::cellid::CellID(place.s2_cell_id)); - country(ll.lng.deg(), ll.lat.deg()) -} - /// The Wikidata IDs in `osm`'s brand/operator/network tags, with /// duplicates, and their [BrandTotals]. Visits every feature in the /// index, in parallel; tag keys are compared as string pool indices, so @@ -349,10 +345,7 @@ mod tests { assert_eq!(features.lines().count(), 1 + 2, "{features}"); assert!(features.contains("Q680727\t\tshop=supermarket\t1\n")); let names = std::fs::read_to_string(brand_totals::names_path(workdir.path()))?; - assert!( - names.contains("Q379911\tCH\tbrand\tDenner\t0\t1\n"), - "{names}" - ); + assert!(names.contains("Q379911\tbrand\tDenner\t0\t1\n"), "{names}"); // Calling again should reuse the existing file, not fail or // recompute.