From 87b19e217e0d25bbe73413c0c17844ce0e015ebe Mon Sep 17 00:00:00 2001 From: Mathious6 <91007976+Mathious6@users.noreply.github.com> Date: Mon, 31 Aug 2026 14:36:25 +0200 Subject: [PATCH 1/3] feat(country): refresh probed subnets in bulk --- AGENTS.md | 5 +- README.md | 7 ++- src-tauri/src/commands.rs | 16 +++-- src-tauri/src/country.rs | 125 ++++++++++++++++++++++++++++++++++---- src-tauri/src/session.rs | 38 +++++++++++- src/lib/import.ts | 1 + src/routes/+page.svelte | 4 +- 7 files changed, 173 insertions(+), 23 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index b4d6c09..315520b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -12,7 +12,8 @@ One window, not resizable. One workflow: 1. Drop a `.txt` file or a folder of `.txt` files, or Open files (`host:port:user:pass` HTTP only). -2. Table of IPv4 `/24` subnets. Country at import. Tags persist by CIDR. +2. Table of IPv4 `/24` subnets. Country lookup batches missing imports and + refreshes every probed subnet. Tags persist by CIDR. Inventory persists locally; Open files appends. Last probe metrics persist until that `/24` gains lines. The last HTTPS target is remembered. 3. Select `/24`s with click, Cmd/Ctrl-click, or Shift-click across the current @@ -58,7 +59,7 @@ asked. | Shell | Tauri 2 | egui, iced, Electron | | UI | Svelte 5 + Tailwind 4 + TanStack Table | router, extra pages, component kits | | Probe | tokio + HTTP CONNECT + rustls (ring) + hyper HTTP/1 | reqwest, libcurl, isahc | -| Country | `GET https://api.country.is/{ip}` on one IP per `/24`, direct (`ureq`) | through-proxy lookup, GeoLite in-repo | +| Country | `POST https://api.country.is/` in direct (`ureq`) batches of at most 100 IPs | through-proxy lookup, GeoLite in-repo | | Tags | local JSON keyed by CIDR | filename-as-identity | Connect = TCP to the proxy + CONNECT `200`, before origin TLS. diff --git a/README.md b/README.md index f10c257..540fb5b 100644 --- a/README.md +++ b/README.md @@ -64,9 +64,10 @@ usually already have it. Windows installers are unsigned. lines and `#` comments are ignored. Anything else is skipped. Imported lines, including credentials, are stored unencrypted in the app's local data directory. -3. Rows appear immediately, one per IPv4 `/24`. Country is looked up from - the listed IP at import through [country.is](https://country.is), not - during the speed test. This sends one IP from each `/24` to that service. +3. Rows appear immediately, one per IPv4 `/24`. Country is looked up in + batches from one listed IP per missing `/24` through + [country.is](https://country.is). Each Probe also refreshes the country + for every `/24` it covers. 4. Tag subnets if you want. Tags are stored on this computer by CIDR and survive the next import. Click a column header to sort. **Filter** sits with Open files, Export, and Probe all. The table shows 15 subnets per diff --git a/src-tauri/src/commands.rs b/src-tauri/src/commands.rs index f3dcc53..2a1fa22 100644 --- a/src-tauri/src/commands.rs +++ b/src-tauri/src/commands.rs @@ -26,6 +26,7 @@ pub fn open_last_target(app: &AppHandle) -> Result { pub struct RunResult { pub completed_at: u64, pub metrics: Vec, + pub countries: HashMap, } #[tauri::command] @@ -51,11 +52,17 @@ pub async fn start_run( } session.resolve_scope(cidrs)? }; + let samples: Vec<_> = buckets + .iter() + .filter_map(|bucket| bucket.proxies.first().map(|proxy| proxy.host)) + .collect(); let window = app.clone(); - let finished = run::probe_session(buckets, target, move |progress: Progress| { + let probe = run::probe_session(buckets, target, move |progress: Progress| { let _ = window.emit(PROGRESS_EVENT, progress); - }) - .await; + }); + let countries = tauri::async_runtime::spawn_blocking(move || crate::country::lookup(&samples)); + let (finished, countries) = tokio::join!(probe, countries); + let countries = countries.map_err(|err| err.to_string())?; let completed_at = SystemTime::now() .duration_since(UNIX_EPOCH) .map(|value| value.as_millis() as u64) @@ -67,7 +74,7 @@ pub async fn start_run( .iter() .map(|(cidr, metrics)| (cidr.clone(), stored_metrics(metrics))) .collect(); - candidate.record_probes(completed_at, &stored); + candidate.record_probes(completed_at, &stored, &countries); let snapshot = candidate.snapshot(); inventory.0.save(&snapshot)?; *session = candidate; @@ -77,6 +84,7 @@ pub async fn start_run( Ok(RunResult { completed_at, metrics, + countries, }) } diff --git a/src-tauri/src/country.rs b/src-tauri/src/country.rs index 7cbc4bc..e0f4e4a 100644 --- a/src-tauri/src/country.rs +++ b/src-tauri/src/country.rs @@ -1,33 +1,69 @@ -use std::collections::HashMap; +use std::collections::{BTreeSet, HashMap, HashSet}; use std::net::Ipv4Addr; -use std::time::Duration; +use std::thread; +use std::time::{Duration, Instant}; use crate::split::Subnet; -const ENDPOINT: &str = "https://api.country.is"; +const ENDPOINT: &str = "https://api.country.is/"; const TIMEOUT: Duration = Duration::from_secs(8); +const BATCH_SIZE: usize = 100; +const REQUEST_INTERVAL: Duration = Duration::from_millis(100); #[derive(serde::Deserialize)] struct Lookup { + ip: String, country: Option, } pub fn lookup(ips: &[Ipv4Addr]) -> HashMap { + lookup_with(ips, fetch) +} + +fn lookup_with(ips: &[Ipv4Addr], mut send: F) -> HashMap +where + F: FnMut(&[Ipv4Addr]) -> Option>, +{ + let ips: Vec<_> = ips + .iter() + .copied() + .collect::>() + .into_iter() + .collect(); + let requested: HashSet<_> = ips.iter().copied().collect(); let mut countries = HashMap::new(); - for ip in ips { - if let Some(row) = fetch(*ip) { - if let Some(code) = normalize(row.country.as_deref()) { - countries.insert(Subnet::from_host(*ip).cidr(), code); + let mut last_request: Option = None; + + for batch in ips.chunks(BATCH_SIZE) { + if let Some(last_request) = last_request { + let elapsed = last_request.elapsed(); + if elapsed < REQUEST_INTERVAL { + thread::sleep(REQUEST_INTERVAL - elapsed); + } + } + last_request = Some(Instant::now()); + let Some(rows) = send(batch) else { + continue; + }; + for row in rows { + let Ok(ip) = row.ip.parse::() else { + continue; + }; + if requested.contains(&ip) { + if let Some(country) = normalize(row.country.as_deref()) { + countries.insert(Subnet::from_host(ip).cidr(), country); + } } } } countries } -fn fetch(ip: Ipv4Addr) -> Option { - ureq::get(&format!("{ENDPOINT}/{ip}")) +fn fetch(ips: &[Ipv4Addr]) -> Option> { + let ips: Vec<_> = ips.iter().map(Ipv4Addr::to_string).collect(); + ureq::post(ENDPOINT) .timeout(TIMEOUT) - .call() + .send_json(ips) .ok()? .into_json() .ok() @@ -35,7 +71,7 @@ fn fetch(ip: Ipv4Addr) -> Option { fn normalize(value: Option<&str>) -> Option { let code = value?; - if code.len() == 2 && code.bytes().all(|b| b.is_ascii_alphabetic()) { + if code.len() == 2 && code.bytes().all(|byte| byte.is_ascii_alphabetic()) { Some(code.to_ascii_uppercase()) } else { None @@ -46,6 +82,73 @@ fn normalize(value: Option<&str>) -> Option { mod tests { use super::*; + fn ips(count: usize) -> Vec { + (0..count) + .map(|index| Ipv4Addr::new(10, (index / 256) as u8, (index % 256) as u8, 1)) + .collect() + } + + #[test] + fn lookup_chunks_batches_at_one_hundred() { + for (count, sizes) in [ + (100, vec![100]), + (101, vec![100, 1]), + (250, vec![100, 100, 50]), + ] { + let mut sent = Vec::new(); + lookup_with(&ips(count), |batch| { + sent.push(batch.len()); + Some(Vec::new()) + }); + assert_eq!(sent, sizes); + } + } + + #[test] + fn lookup_deduplicates_ips_and_maps_returned_ips() { + let first: Ipv4Addr = "192.0.2.10".parse().unwrap(); + let second: Ipv4Addr = "198.51.100.20".parse().unwrap(); + let mut sent = Vec::new(); + let countries = lookup_with(&[first, second, first], |batch| { + sent.extend_from_slice(batch); + Some(vec![ + Lookup { + ip: second.to_string(), + country: Some("de".into()), + }, + Lookup { + ip: first.to_string(), + country: Some("fr".into()), + }, + ]) + }); + assert_eq!(sent.len(), 2); + assert_eq!(countries.get("192.0.2.0/24"), Some(&"FR".into())); + assert_eq!(countries.get("198.51.100.0/24"), Some(&"DE".into())); + } + + #[test] + fn lookup_ignores_omitted_malformed_and_invalid_rows() { + let requested: Ipv4Addr = "192.0.2.10".parse().unwrap(); + let countries = lookup_with(&[requested], |_| { + Some(vec![ + Lookup { + ip: "not-an-ip".into(), + country: Some("FR".into()), + }, + Lookup { + ip: "198.51.100.20".into(), + country: Some("DE".into()), + }, + Lookup { + ip: requested.to_string(), + country: Some("FRA".into()), + }, + ]) + }); + assert!(countries.is_empty()); + } + #[test] fn normalize_uppercases_iso_alpha2() { assert_eq!(normalize(Some("fr")), Some("FR".into())); diff --git a/src-tauri/src/session.rs b/src-tauri/src/session.rs index 616fae2..9eb79d2 100644 --- a/src-tauri/src/session.rs +++ b/src-tauri/src/session.rs @@ -97,10 +97,23 @@ impl Session { } } - pub fn record_probes(&mut self, at: u64, metrics: &HashMap) { + pub fn record_probes( + &mut self, + at: u64, + metrics: &HashMap, + countries: &HashMap, + ) { for (cidr, item) in metrics { self.record_probe(cidr, at, Some(item.clone())); } + for (cidr, country) in countries { + let Some(subnet) = Subnet::parse_cidr(cidr) else { + continue; + }; + if let Some(bucket) = self.buckets.get_mut(&subnet) { + bucket.country = Some(country.clone()); + } + } } pub fn snapshot(&self) -> Vec { @@ -303,7 +316,7 @@ mod tests { let mut metrics = HashMap::new(); metrics.insert("192.0.2.0/24".into(), sample_metrics(1)); metrics.insert("203.0.113.0/24".into(), sample_metrics(8)); - session.record_probes(42, &metrics); + session.record_probes(42, &metrics, &HashMap::new()); let snapshot = session.snapshot(); let first = snapshot .iter() @@ -317,6 +330,27 @@ mod tests { assert_eq!(second.last_probe, None); } + #[test] + fn record_probes_refreshes_valid_countries_and_keeps_missing_ones() { + let mut session = Session::new(); + let mut initial = HashMap::new(); + initial.insert("192.0.2.0/24".into(), "FR".into()); + initial.insert("198.51.100.0/24".into(), "DE".into()); + session.merge( + by_slash24(vec![proxy("192.0.2.10"), proxy("198.51.100.2")]), + &initial, + ); + let mut metrics = HashMap::new(); + metrics.insert("192.0.2.0/24".into(), sample_metrics(1)); + metrics.insert("198.51.100.0/24".into(), sample_metrics(1)); + let mut countries = HashMap::new(); + countries.insert("192.0.2.0/24".into(), "US".into()); + session.record_probes(42, &metrics, &countries); + let snapshot = session.snapshot(); + assert_eq!(snapshot[0].country.as_deref(), Some("US")); + assert_eq!(snapshot[1].country.as_deref(), Some("DE")); + } + #[test] fn resolve_scope_returns_all_buckets() { let mut session = Session::new(); diff --git a/src/lib/import.ts b/src/lib/import.ts index 5dcac38..8255b84 100644 --- a/src/lib/import.ts +++ b/src/lib/import.ts @@ -37,6 +37,7 @@ export type Progress = { export type RunResult = { completedAt: number; metrics: Metrics[]; + countries: Record; }; export function emptyMetrics(): Pick< diff --git a/src/routes/+page.svelte b/src/routes/+page.svelte index a306ae7..1ea9010 100644 --- a/src/routes/+page.svelte +++ b/src/routes/+page.svelte @@ -298,7 +298,9 @@ const byCidr = new Map(probed.metrics.map((metrics) => [metrics.cidr, metrics])); rows = rows.map((row) => { const metrics = byCidr.get(row.cidr); - return metrics ? withMetrics(row, metrics, probed.completedAt) : row; + const country = probed.countries[row.cidr]; + const next = metrics ? withMetrics(row, metrics, probed.completedAt) : row; + return country ? { ...next, country } : next; }); } catch (error) { try { From d114e10418680729d744c3eb38847af022371047 Mon Sep 17 00:00:00 2001 From: Mathious6 <91007976+Mathious6@users.noreply.github.com> Date: Mon, 31 Aug 2026 15:41:12 +0200 Subject: [PATCH 2/3] fix(country): retry bulk lookup failures --- src-tauri/src/country.rs | 70 ++++++++++++++++++++++++++++++++-------- 1 file changed, 57 insertions(+), 13 deletions(-) diff --git a/src-tauri/src/country.rs b/src-tauri/src/country.rs index e0f4e4a..8ce3a69 100644 --- a/src-tauri/src/country.rs +++ b/src-tauri/src/country.rs @@ -9,6 +9,8 @@ const ENDPOINT: &str = "https://api.country.is/"; const TIMEOUT: Duration = Duration::from_secs(8); const BATCH_SIZE: usize = 100; const REQUEST_INTERVAL: Duration = Duration::from_millis(100); +const RETRY_DELAY: Duration = Duration::from_millis(250); +const ATTEMPTS: usize = 2; #[derive(serde::Deserialize)] struct Lookup { @@ -22,7 +24,7 @@ pub fn lookup(ips: &[Ipv4Addr]) -> HashMap { fn lookup_with(ips: &[Ipv4Addr], mut send: F) -> HashMap where - F: FnMut(&[Ipv4Addr]) -> Option>, + F: FnMut(&[Ipv4Addr]) -> Result, ()>, { let ips: Vec<_> = ips .iter() @@ -35,14 +37,20 @@ where let mut last_request: Option = None; for batch in ips.chunks(BATCH_SIZE) { - if let Some(last_request) = last_request { - let elapsed = last_request.elapsed(); - if elapsed < REQUEST_INTERVAL { - thread::sleep(REQUEST_INTERVAL - elapsed); + let mut rows = None; + for attempt in 0..ATTEMPTS { + wait_for_request(last_request); + last_request = Some(Instant::now()); + match send(batch) { + Ok(value) => { + rows = Some(value); + break; + } + Err(()) if attempt + 1 < ATTEMPTS => thread::sleep(RETRY_DELAY), + Err(()) => {} } } - last_request = Some(Instant::now()); - let Some(rows) = send(batch) else { + let Some(rows) = rows else { continue; }; for row in rows { @@ -59,14 +67,23 @@ where countries } -fn fetch(ips: &[Ipv4Addr]) -> Option> { +fn wait_for_request(last_request: Option) { + if let Some(last_request) = last_request { + let elapsed = last_request.elapsed(); + if elapsed < REQUEST_INTERVAL { + thread::sleep(REQUEST_INTERVAL - elapsed); + } + } +} + +fn fetch(ips: &[Ipv4Addr]) -> Result, ()> { let ips: Vec<_> = ips.iter().map(Ipv4Addr::to_string).collect(); ureq::post(ENDPOINT) .timeout(TIMEOUT) .send_json(ips) - .ok()? + .map_err(|_| ())? .into_json() - .ok() + .map_err(|_| ()) } fn normalize(value: Option<&str>) -> Option { @@ -98,7 +115,7 @@ mod tests { let mut sent = Vec::new(); lookup_with(&ips(count), |batch| { sent.push(batch.len()); - Some(Vec::new()) + Ok(Vec::new()) }); assert_eq!(sent, sizes); } @@ -111,7 +128,7 @@ mod tests { let mut sent = Vec::new(); let countries = lookup_with(&[first, second, first], |batch| { sent.extend_from_slice(batch); - Some(vec![ + Ok(vec![ Lookup { ip: second.to_string(), country: Some("de".into()), @@ -131,7 +148,7 @@ mod tests { fn lookup_ignores_omitted_malformed_and_invalid_rows() { let requested: Ipv4Addr = "192.0.2.10".parse().unwrap(); let countries = lookup_with(&[requested], |_| { - Some(vec![ + Ok(vec![ Lookup { ip: "not-an-ip".into(), country: Some("FR".into()), @@ -149,6 +166,33 @@ mod tests { assert!(countries.is_empty()); } + #[test] + fn lookup_retries_failed_batches() { + let requested: Ipv4Addr = "151.242.94.0".parse().unwrap(); + let mut attempts = 0; + let countries = lookup_with(&[requested], |_| { + attempts += 1; + if attempts == 1 { + Err(()) + } else { + Ok(vec![Lookup { + ip: requested.to_string(), + country: Some("DE".into()), + }]) + } + }); + assert_eq!(attempts, 2); + assert_eq!(countries.get("151.242.94.0/24"), Some(&"DE".into())); + } + + #[test] + fn lookup_decodes_bulk_response() { + let rows: Vec = + serde_json::from_str(r#"[{"ip":"151.242.94.0","country":"DE"}]"#).unwrap(); + assert_eq!(rows[0].ip, "151.242.94.0"); + assert_eq!(rows[0].country.as_deref(), Some("DE")); + } + #[test] fn normalize_uppercases_iso_alpha2() { assert_eq!(normalize(Some("fr")), Some("FR".into())); From 44e5c6596d5fc78fdcce3960b050dd289451dc91 Mon Sep 17 00:00:00 2001 From: Mathious6 <91007976+Mathious6@users.noreply.github.com> Date: Mon, 31 Aug 2026 16:26:16 +0200 Subject: [PATCH 3/3] fix(country): refresh before probing --- AGENTS.md | 2 +- README.md | 4 ++-- src-tauri/src/commands.rs | 19 +++++++++++++------ src-tauri/src/session.rs | 19 +++++++------------ 4 files changed, 23 insertions(+), 21 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 315520b..6f6fd83 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -13,7 +13,7 @@ One window, not resizable. One workflow: 1. Drop a `.txt` file or a folder of `.txt` files, or Open files (`host:port:user:pass` HTTP only). 2. Table of IPv4 `/24` subnets. Country lookup batches missing imports and - refreshes every probed subnet. Tags persist by CIDR. + refreshes every probed subnet before speed measurements. Tags persist by CIDR. Inventory persists locally; Open files appends. Last probe metrics persist until that `/24` gains lines. The last HTTPS target is remembered. 3. Select `/24`s with click, Cmd/Ctrl-click, or Shift-click across the current diff --git a/README.md b/README.md index 540fb5b..aa93dd4 100644 --- a/README.md +++ b/README.md @@ -66,8 +66,8 @@ usually already have it. Windows installers are unsigned. data directory. 3. Rows appear immediately, one per IPv4 `/24`. Country is looked up in batches from one listed IP per missing `/24` through - [country.is](https://country.is). Each Probe also refreshes the country - for every `/24` it covers. + [country.is](https://country.is). Each Probe refreshes and saves the country + for every `/24` it covers before starting the speed measurements. 4. Tag subnets if you want. Tags are stored on this computer by CIDR and survive the next import. Click a column header to sort. **Filter** sits with Open files, Export, and Probe all. The table shows 15 subnets per diff --git a/src-tauri/src/commands.rs b/src-tauri/src/commands.rs index 2a1fa22..5789e12 100644 --- a/src-tauri/src/commands.rs +++ b/src-tauri/src/commands.rs @@ -56,13 +56,20 @@ pub async fn start_run( .iter() .filter_map(|bucket| bucket.proxies.first().map(|proxy| proxy.host)) .collect(); + let countries = tauri::async_runtime::spawn_blocking(move || crate::country::lookup(&samples)); + let countries = countries.await.map_err(|err| err.to_string())?; + if !countries.is_empty() { + let mut session = session.0.lock().map_err(|err| err.to_string())?; + let mut candidate = session.clone(); + candidate.record_countries(&countries); + inventory.0.save(&candidate.snapshot())?; + *session = candidate; + } let window = app.clone(); - let probe = run::probe_session(buckets, target, move |progress: Progress| { + let finished = run::probe_session(buckets, target, move |progress: Progress| { let _ = window.emit(PROGRESS_EVENT, progress); - }); - let countries = tauri::async_runtime::spawn_blocking(move || crate::country::lookup(&samples)); - let (finished, countries) = tokio::join!(probe, countries); - let countries = countries.map_err(|err| err.to_string())?; + }) + .await; let completed_at = SystemTime::now() .duration_since(UNIX_EPOCH) .map(|value| value.as_millis() as u64) @@ -74,7 +81,7 @@ pub async fn start_run( .iter() .map(|(cidr, metrics)| (cidr.clone(), stored_metrics(metrics))) .collect(); - candidate.record_probes(completed_at, &stored, &countries); + candidate.record_probes(completed_at, &stored); let snapshot = candidate.snapshot(); inventory.0.save(&snapshot)?; *session = candidate; diff --git a/src-tauri/src/session.rs b/src-tauri/src/session.rs index 9eb79d2..c569511 100644 --- a/src-tauri/src/session.rs +++ b/src-tauri/src/session.rs @@ -97,15 +97,13 @@ impl Session { } } - pub fn record_probes( - &mut self, - at: u64, - metrics: &HashMap, - countries: &HashMap, - ) { + pub fn record_probes(&mut self, at: u64, metrics: &HashMap) { for (cidr, item) in metrics { self.record_probe(cidr, at, Some(item.clone())); } + } + + pub fn record_countries(&mut self, countries: &HashMap) { for (cidr, country) in countries { let Some(subnet) = Subnet::parse_cidr(cidr) else { continue; @@ -316,7 +314,7 @@ mod tests { let mut metrics = HashMap::new(); metrics.insert("192.0.2.0/24".into(), sample_metrics(1)); metrics.insert("203.0.113.0/24".into(), sample_metrics(8)); - session.record_probes(42, &metrics, &HashMap::new()); + session.record_probes(42, &metrics); let snapshot = session.snapshot(); let first = snapshot .iter() @@ -331,7 +329,7 @@ mod tests { } #[test] - fn record_probes_refreshes_valid_countries_and_keeps_missing_ones() { + fn record_countries_refreshes_valid_values_and_keeps_missing_ones() { let mut session = Session::new(); let mut initial = HashMap::new(); initial.insert("192.0.2.0/24".into(), "FR".into()); @@ -340,12 +338,9 @@ mod tests { by_slash24(vec![proxy("192.0.2.10"), proxy("198.51.100.2")]), &initial, ); - let mut metrics = HashMap::new(); - metrics.insert("192.0.2.0/24".into(), sample_metrics(1)); - metrics.insert("198.51.100.0/24".into(), sample_metrics(1)); let mut countries = HashMap::new(); countries.insert("192.0.2.0/24".into(), "US".into()); - session.record_probes(42, &metrics, &countries); + session.record_countries(&countries); let snapshot = session.snapshot(); assert_eq!(snapshot[0].country.as_deref(), Some("US")); assert_eq!(snapshot[1].country.as_deref(), Some("DE"));