diff --git a/AGENTS.md b/AGENTS.md index b4d6c09..6f6fd83 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -12,7 +12,8 @@ One window, not resizable. One workflow: 1. Drop a `.txt` file or a folder of `.txt` files, or Open files (`host:port:user:pass` HTTP only). -2. Table of IPv4 `/24` subnets. Country at import. Tags persist by CIDR. +2. Table of IPv4 `/24` subnets. Country lookup batches missing imports and + refreshes every probed subnet before speed measurements. Tags persist by CIDR. Inventory persists locally; Open files appends. Last probe metrics persist until that `/24` gains lines. The last HTTPS target is remembered. 3. Select `/24`s with click, Cmd/Ctrl-click, or Shift-click across the current @@ -58,7 +59,7 @@ asked. | Shell | Tauri 2 | egui, iced, Electron | | UI | Svelte 5 + Tailwind 4 + TanStack Table | router, extra pages, component kits | | Probe | tokio + HTTP CONNECT + rustls (ring) + hyper HTTP/1 | reqwest, libcurl, isahc | -| Country | `GET https://api.country.is/{ip}` on one IP per `/24`, direct (`ureq`) | through-proxy lookup, GeoLite in-repo | +| Country | `POST https://api.country.is/` in direct (`ureq`) batches of at most 100 IPs | through-proxy lookup, GeoLite in-repo | | Tags | local JSON keyed by CIDR | filename-as-identity | Connect = TCP to the proxy + CONNECT `200`, before origin TLS. diff --git a/README.md b/README.md index f10c257..aa93dd4 100644 --- a/README.md +++ b/README.md @@ -64,9 +64,10 @@ usually already have it. Windows installers are unsigned. lines and `#` comments are ignored. Anything else is skipped. Imported lines, including credentials, are stored unencrypted in the app's local data directory. -3. Rows appear immediately, one per IPv4 `/24`. Country is looked up from - the listed IP at import through [country.is](https://country.is), not - during the speed test. This sends one IP from each `/24` to that service. +3. Rows appear immediately, one per IPv4 `/24`. Country is looked up in + batches from one listed IP per missing `/24` through + [country.is](https://country.is). Each Probe refreshes and saves the country + for every `/24` it covers before starting the speed measurements. 4. Tag subnets if you want. Tags are stored on this computer by CIDR and survive the next import. Click a column header to sort. **Filter** sits with Open files, Export, and Probe all. The table shows 15 subnets per diff --git a/src-tauri/src/commands.rs b/src-tauri/src/commands.rs index f3dcc53..5789e12 100644 --- a/src-tauri/src/commands.rs +++ b/src-tauri/src/commands.rs @@ -26,6 +26,7 @@ pub fn open_last_target(app: &AppHandle) -> Result { pub struct RunResult { pub completed_at: u64, pub metrics: Vec, + pub countries: HashMap, } #[tauri::command] @@ -51,6 +52,19 @@ pub async fn start_run( } session.resolve_scope(cidrs)? }; + let samples: Vec<_> = buckets + .iter() + .filter_map(|bucket| bucket.proxies.first().map(|proxy| proxy.host)) + .collect(); + let countries = tauri::async_runtime::spawn_blocking(move || crate::country::lookup(&samples)); + let countries = countries.await.map_err(|err| err.to_string())?; + if !countries.is_empty() { + let mut session = session.0.lock().map_err(|err| err.to_string())?; + let mut candidate = session.clone(); + candidate.record_countries(&countries); + inventory.0.save(&candidate.snapshot())?; + *session = candidate; + } let window = app.clone(); let finished = run::probe_session(buckets, target, move |progress: Progress| { let _ = window.emit(PROGRESS_EVENT, progress); @@ -77,6 +91,7 @@ pub async fn start_run( Ok(RunResult { completed_at, metrics, + countries, }) } diff --git a/src-tauri/src/country.rs b/src-tauri/src/country.rs index 7cbc4bc..8ce3a69 100644 --- a/src-tauri/src/country.rs +++ b/src-tauri/src/country.rs @@ -1,41 +1,94 @@ -use std::collections::HashMap; +use std::collections::{BTreeSet, HashMap, HashSet}; use std::net::Ipv4Addr; -use std::time::Duration; +use std::thread; +use std::time::{Duration, Instant}; use crate::split::Subnet; -const ENDPOINT: &str = "https://api.country.is"; +const ENDPOINT: &str = "https://api.country.is/"; const TIMEOUT: Duration = Duration::from_secs(8); +const BATCH_SIZE: usize = 100; +const REQUEST_INTERVAL: Duration = Duration::from_millis(100); +const RETRY_DELAY: Duration = Duration::from_millis(250); +const ATTEMPTS: usize = 2; #[derive(serde::Deserialize)] struct Lookup { + ip: String, country: Option, } pub fn lookup(ips: &[Ipv4Addr]) -> HashMap { + lookup_with(ips, fetch) +} + +fn lookup_with(ips: &[Ipv4Addr], mut send: F) -> HashMap +where + F: FnMut(&[Ipv4Addr]) -> Result, ()>, +{ + let ips: Vec<_> = ips + .iter() + .copied() + .collect::>() + .into_iter() + .collect(); + let requested: HashSet<_> = ips.iter().copied().collect(); let mut countries = HashMap::new(); - for ip in ips { - if let Some(row) = fetch(*ip) { - if let Some(code) = normalize(row.country.as_deref()) { - countries.insert(Subnet::from_host(*ip).cidr(), code); + let mut last_request: Option = None; + + for batch in ips.chunks(BATCH_SIZE) { + let mut rows = None; + for attempt in 0..ATTEMPTS { + wait_for_request(last_request); + last_request = Some(Instant::now()); + match send(batch) { + Ok(value) => { + rows = Some(value); + break; + } + Err(()) if attempt + 1 < ATTEMPTS => thread::sleep(RETRY_DELAY), + Err(()) => {} + } + } + let Some(rows) = rows else { + continue; + }; + for row in rows { + let Ok(ip) = row.ip.parse::() else { + continue; + }; + if requested.contains(&ip) { + if let Some(country) = normalize(row.country.as_deref()) { + countries.insert(Subnet::from_host(ip).cidr(), country); + } } } } countries } -fn fetch(ip: Ipv4Addr) -> Option { - ureq::get(&format!("{ENDPOINT}/{ip}")) +fn wait_for_request(last_request: Option) { + if let Some(last_request) = last_request { + let elapsed = last_request.elapsed(); + if elapsed < REQUEST_INTERVAL { + thread::sleep(REQUEST_INTERVAL - elapsed); + } + } +} + +fn fetch(ips: &[Ipv4Addr]) -> Result, ()> { + let ips: Vec<_> = ips.iter().map(Ipv4Addr::to_string).collect(); + ureq::post(ENDPOINT) .timeout(TIMEOUT) - .call() - .ok()? + .send_json(ips) + .map_err(|_| ())? .into_json() - .ok() + .map_err(|_| ()) } fn normalize(value: Option<&str>) -> Option { let code = value?; - if code.len() == 2 && code.bytes().all(|b| b.is_ascii_alphabetic()) { + if code.len() == 2 && code.bytes().all(|byte| byte.is_ascii_alphabetic()) { Some(code.to_ascii_uppercase()) } else { None @@ -46,6 +99,100 @@ fn normalize(value: Option<&str>) -> Option { mod tests { use super::*; + fn ips(count: usize) -> Vec { + (0..count) + .map(|index| Ipv4Addr::new(10, (index / 256) as u8, (index % 256) as u8, 1)) + .collect() + } + + #[test] + fn lookup_chunks_batches_at_one_hundred() { + for (count, sizes) in [ + (100, vec![100]), + (101, vec![100, 1]), + (250, vec![100, 100, 50]), + ] { + let mut sent = Vec::new(); + lookup_with(&ips(count), |batch| { + sent.push(batch.len()); + Ok(Vec::new()) + }); + assert_eq!(sent, sizes); + } + } + + #[test] + fn lookup_deduplicates_ips_and_maps_returned_ips() { + let first: Ipv4Addr = "192.0.2.10".parse().unwrap(); + let second: Ipv4Addr = "198.51.100.20".parse().unwrap(); + let mut sent = Vec::new(); + let countries = lookup_with(&[first, second, first], |batch| { + sent.extend_from_slice(batch); + Ok(vec![ + Lookup { + ip: second.to_string(), + country: Some("de".into()), + }, + Lookup { + ip: first.to_string(), + country: Some("fr".into()), + }, + ]) + }); + assert_eq!(sent.len(), 2); + assert_eq!(countries.get("192.0.2.0/24"), Some(&"FR".into())); + assert_eq!(countries.get("198.51.100.0/24"), Some(&"DE".into())); + } + + #[test] + fn lookup_ignores_omitted_malformed_and_invalid_rows() { + let requested: Ipv4Addr = "192.0.2.10".parse().unwrap(); + let countries = lookup_with(&[requested], |_| { + Ok(vec![ + Lookup { + ip: "not-an-ip".into(), + country: Some("FR".into()), + }, + Lookup { + ip: "198.51.100.20".into(), + country: Some("DE".into()), + }, + Lookup { + ip: requested.to_string(), + country: Some("FRA".into()), + }, + ]) + }); + assert!(countries.is_empty()); + } + + #[test] + fn lookup_retries_failed_batches() { + let requested: Ipv4Addr = "151.242.94.0".parse().unwrap(); + let mut attempts = 0; + let countries = lookup_with(&[requested], |_| { + attempts += 1; + if attempts == 1 { + Err(()) + } else { + Ok(vec![Lookup { + ip: requested.to_string(), + country: Some("DE".into()), + }]) + } + }); + assert_eq!(attempts, 2); + assert_eq!(countries.get("151.242.94.0/24"), Some(&"DE".into())); + } + + #[test] + fn lookup_decodes_bulk_response() { + let rows: Vec = + serde_json::from_str(r#"[{"ip":"151.242.94.0","country":"DE"}]"#).unwrap(); + assert_eq!(rows[0].ip, "151.242.94.0"); + assert_eq!(rows[0].country.as_deref(), Some("DE")); + } + #[test] fn normalize_uppercases_iso_alpha2() { assert_eq!(normalize(Some("fr")), Some("FR".into())); diff --git a/src-tauri/src/session.rs b/src-tauri/src/session.rs index 616fae2..c569511 100644 --- a/src-tauri/src/session.rs +++ b/src-tauri/src/session.rs @@ -103,6 +103,17 @@ impl Session { } } + pub fn record_countries(&mut self, countries: &HashMap) { + for (cidr, country) in countries { + let Some(subnet) = Subnet::parse_cidr(cidr) else { + continue; + }; + if let Some(bucket) = self.buckets.get_mut(&subnet) { + bucket.country = Some(country.clone()); + } + } + } + pub fn snapshot(&self) -> Vec { self.buckets.values().cloned().collect() } @@ -317,6 +328,24 @@ mod tests { assert_eq!(second.last_probe, None); } + #[test] + fn record_countries_refreshes_valid_values_and_keeps_missing_ones() { + let mut session = Session::new(); + let mut initial = HashMap::new(); + initial.insert("192.0.2.0/24".into(), "FR".into()); + initial.insert("198.51.100.0/24".into(), "DE".into()); + session.merge( + by_slash24(vec![proxy("192.0.2.10"), proxy("198.51.100.2")]), + &initial, + ); + let mut countries = HashMap::new(); + countries.insert("192.0.2.0/24".into(), "US".into()); + session.record_countries(&countries); + let snapshot = session.snapshot(); + assert_eq!(snapshot[0].country.as_deref(), Some("US")); + assert_eq!(snapshot[1].country.as_deref(), Some("DE")); + } + #[test] fn resolve_scope_returns_all_buckets() { let mut session = Session::new(); diff --git a/src/lib/import.ts b/src/lib/import.ts index 5dcac38..8255b84 100644 --- a/src/lib/import.ts +++ b/src/lib/import.ts @@ -37,6 +37,7 @@ export type Progress = { export type RunResult = { completedAt: number; metrics: Metrics[]; + countries: Record; }; export function emptyMetrics(): Pick< diff --git a/src/routes/+page.svelte b/src/routes/+page.svelte index a306ae7..1ea9010 100644 --- a/src/routes/+page.svelte +++ b/src/routes/+page.svelte @@ -298,7 +298,9 @@ const byCidr = new Map(probed.metrics.map((metrics) => [metrics.cidr, metrics])); rows = rows.map((row) => { const metrics = byCidr.get(row.cidr); - return metrics ? withMetrics(row, metrics, probed.completedAt) : row; + const country = probed.countries[row.cidr]; + const next = metrics ? withMetrics(row, metrics, probed.completedAt) : row; + return country ? { ...next, country } : next; }); } catch (error) { try {