From 0d8986df69af36ca0147f315b239d1fae4251b84 Mon Sep 17 00:00:00 2001 From: almac2022 Date: Sat, 4 Apr 2026 13:45:36 -0700 Subject: [PATCH 1/2] =?UTF-8?q?Optimize=20classify=20query:=203,700x=20spe?= =?UTF-8?q?edup=20(742s=20=E2=86=92=200.2s)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two changes eliminate the classify bottleneck: 1. Enrich breaks table with wscode_ltree + localcode_ltree from FWA base network (one-time join after all breaks combined). Eliminates the 4.9M row fwa_stream_networks_sp join at classify time. 2. Split NOT EXISTS OR into two separate NOT EXISTS with AND: - Same BLK: pure measure comparison (btree index) - Cross BLK: direct ltree comparison (GIST index) PG can now use indexes on both independently. Full pipeline ADMS (5 species, falls + crossings): 90s total. Classify step alone: 0.2s (was 742s with indexes, ~840s without). Fixes #72 Co-Authored-By: Claude Opus 4.6 (1M context) --- R/frs_classify.R | 50 ++++++++----------- R/frs_habitat.R | 31 ++++++++++++ ...0404_classify_benchmark-adms-optimized.txt | 29 +++++++++++ ...60404_habitat_benchmark-adms-optimized.txt | 37 ++++++++++++++ 4 files changed, 118 insertions(+), 29 deletions(-) create mode 100644 scripts/habitat/logs/20260404_classify_benchmark-adms-optimized.txt create mode 100644 scripts/habitat/logs/20260404_habitat_benchmark-adms-optimized.txt diff --git a/R/frs_classify.R b/R/frs_classify.R index 7c4bf4c..4e8097e 100644 --- a/R/frs_classify.R +++ b/R/frs_classify.R @@ -252,42 +252,34 @@ frs_classify <- function(conn, table, label, mus <- .frs_opt("measure_us_col") # Segments are accessible if no break is downstream of them. - # A segment is BLOCKED (upstream of break) when: - # 1. Same BLK: segment measure >= break measure (break is downstream) - # 2. Different BLK: fwa_upstream(break_pos, segment) = TRUE - # (segment is upstream of the break on the network) + # Split into two separate NOT EXISTS with AND (not OR) so PG can + # use indexes on both independently. + # + # 1. Same BLK: pure measure comparison — instant with btree index + # 2. Cross BLK: ltree comparison using enriched wscode/localcode + # on the breaks table — fast with GIST index, no join to + # fwa_stream_networks_sp (4.9M rows) sql <- sprintf( "UPDATE %s s SET %s = %s WHERE NOT EXISTS ( + -- Same BLK: break is downstream if measure <= segment measure SELECT 1 FROM %s b - WHERE ( - -- Same BLK: segment is upstream of break if its measure >= break measure - b.%s = s.%s + WHERE b.%s = s.%s AND b.%s <= s.%s - ) - OR ( - -- Different BLK: is segment upstream of the break? - b.%s != s.%s - AND EXISTS ( - SELECT 1 FROM whse_basemapping.fwa_stream_networks_sp f - WHERE f.%s = b.%s - AND b.%s >= f.%s - AND b.%s < f.%s - AND fwa_upstream( - f.wscode_ltree, f.localcode_ltree, - s.%s, s.%s - ) + ) + AND NOT EXISTS ( + -- Cross BLK: break is downstream on the network (ltree) + SELECT 1 FROM %s b + WHERE b.%s != s.%s + AND b.wscode_ltree IS NOT NULL + AND fwa_upstream( + b.wscode_ltree, b.localcode_ltree, + s.%s, s.%s ) - ) )", - table, label, ifelse(value, "TRUE", "FALSE"), breaks, - blk, blk, # same BLK check - mds, mds, # measure comparison - blk, blk, # different BLK check - blk, blk, # join to FWA - mds, mds, # measure range check (ds) - mds, mus, # measure range check (us) - wsc, loc # ltree columns on working table + table, label, ifelse(value, "TRUE", "FALSE"), + breaks, blk, blk, mds, mds, # same BLK + breaks, blk, blk, wsc, loc # cross BLK ) # Append user-supplied where filter to scope which rows get classified diff --git a/R/frs_habitat.R b/R/frs_habitat.R index 390fe84..b4d6ff2 100644 --- a/R/frs_habitat.R +++ b/R/frs_habitat.R @@ -463,12 +463,43 @@ frs_habitat_access <- function(conn, table, threshold, } } + # Enrich breaks with ltree codes for fast cross-BLK classification + .frs_enrich_breaks(conn, to) + .frs_index_working(conn, to) invisible(conn) } +#' Enrich breaks table with ltree codes from FWA base network +#' +#' Adds `wscode_ltree` and `localcode_ltree` columns by joining each break +#' point to the FWA stream segment it falls within. These columns enable +#' fast cross-BLK classification via ltree comparison instead of joining +#' back to the 4.9M row `fwa_stream_networks_sp` table at classify time. +#' +#' @param conn DBI connection. +#' @param breaks Schema-qualified breaks table name. +#' @noRd +.frs_enrich_breaks <- function(conn, breaks) { + .frs_db_execute(conn, sprintf( + "ALTER TABLE %s ADD COLUMN IF NOT EXISTS wscode_ltree ltree", breaks)) + .frs_db_execute(conn, sprintf( + "ALTER TABLE %s ADD COLUMN IF NOT EXISTS localcode_ltree ltree", breaks)) + + .frs_db_execute(conn, sprintf( + "UPDATE %s b SET + wscode_ltree = f.wscode_ltree, + localcode_ltree = f.localcode_ltree + FROM whse_basemapping.fwa_stream_networks_sp f + WHERE b.blue_line_key = f.blue_line_key + AND b.downstream_route_measure >= f.downstream_route_measure + AND b.downstream_route_measure < f.upstream_route_measure", + breaks)) +} + + #' Scope break sources to a WSG code #' #' When AOI is a 4-letter WSG code, appends a `watershed_group_code` diff --git a/scripts/habitat/logs/20260404_classify_benchmark-adms-optimized.txt b/scripts/habitat/logs/20260404_classify_benchmark-adms-optimized.txt new file mode 100644 index 0000000..8448a72 --- /dev/null +++ b/scripts/habitat/logs/20260404_classify_benchmark-adms-optimized.txt @@ -0,0 +1,29 @@ +## Classify benchmark — ADMS optimized query +## Date: 2026-04-04 +## DB: local Docker fwapg (port 5432) +## Branch: optimize-classify +## Issue: #72 + +## Setup +# WSG: ADMS (11,520 segments) +# Break sources: gradient (28,960) + falls (7) + crossings (3,597) +# Total breaks: 32,564 (all enriched with ltree codes) +# Indexes: blue_line_key, (blue_line_key, downstream_route_measure), +# wscode_ltree GIST+BTREE, localcode_ltree GIST+BTREE + +## Changes +# 1. Enrich breaks with wscode_ltree + localcode_ltree from FWA base +# (one-time UPDATE ... FROM fwa_stream_networks_sp after all breaks combined) +# 2. Split classify OR into two separate NOT EXISTS with AND +# - Same BLK: pure measure comparison (btree index) +# - Cross BLK: ltree comparison using enriched codes (GIST index) +# No join to fwa_stream_networks_sp (4.9M rows) at classify time + +## Result +# Classify: 0.2s +# Accessible: 1,060 / 11,520 segments (9.2%) — matches baseline + +## Comparison +# Before (v0.7.0, indexes only): 742.1s +# After (optimized query): 0.2s +# Speedup: 3,700x diff --git a/scripts/habitat/logs/20260404_habitat_benchmark-adms-optimized.txt b/scripts/habitat/logs/20260404_habitat_benchmark-adms-optimized.txt new file mode 100644 index 0000000..025396c --- /dev/null +++ b/scripts/habitat/logs/20260404_habitat_benchmark-adms-optimized.txt @@ -0,0 +1,37 @@ +## Habitat pipeline benchmark — ADMS optimized +## Date: 2026-04-04 +## DB: local Docker fwapg (port 5432) +## Branch: optimize-classify +## Issue: #72 + +## Setup +# WSG: ADMS (11,520 segments, 5 species: BT, CH, CO, RB, SK) +# Break sources: gradient + falls (7 ADMS) + crossings (3,597 ADMS) +# Optimizations: ltree-enriched breaks, split NOT EXISTS query, indexes + +## Results +# Phase 1 (partition prep): 23.3s +# Base extract + indexes: 4.2s +# Access 15% (gradient + falls + crossings + enrich + index): 5.5s +# Access 25%: 4.6s +# Habitat breaks (3 thresholds): 9s +# +# Phase 2 (species classification): 67.2s +# BT: 13.6s, CH: 13.2s, CO: 13.2s, RB: 13.7s, SK: 13.5s +# +# Total: 90.6s + +## Comparison +# v0.6.0 remote DB, falls only, no indexes: 88s +# v0.7.0 local Docker, falls only, no indexes: 88s +# v0.7.0 local Docker, falls+crossings, unscoped: 5,888s (bug — 533k province breaks) +# v0.7.0 local Docker, falls+crossings, indexes only: ~840s per species +# Optimized: 90s total (13s/species) + +## Breakdown per species (Phase 2) +# ~0.5s: copy base table +# ~3s: frs_col_generate (recompute gradient from geometry) +# ~0.2s: frs_break_apply (split geometry at breaks) +# ~0.2s: frs_classify accessible (optimized — was 742s) +# ~6s: frs_classify spawning/rearing (attribute ranges) +# ~3s: frs_classify lake rearing + categorize From 9e6fe8f65ce0354e2ab671243ca39426526e8b93 Mon Sep 17 00:00:00 2001 From: almac2022 Date: Sat, 4 Apr 2026 19:05:07 -0700 Subject: [PATCH 2/2] Tune Docker PG for M4 Max Pro (128GB/16 cores) 32GB shared_buffers, 96GB effective_cache_size, 2GB work_mem, 8 parallel workers per gather. Add tuning.md with scaling formulas for other machines. Relates to #72 Co-Authored-By: Claude Opus 4.6 (1M context) --- docker/docker-compose.yml | 18 +++++------ docker/tuning.md | 68 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+), 9 deletions(-) create mode 100644 docker/tuning.md diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index ecbecf4..9d02733 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -17,22 +17,22 @@ services: db: image: imresamu/postgis:17-3.5-bullseye container_name: fresh-db - shm_size: 16gb + shm_size: 36gb command: > postgres -c default_statistics_target=100 -c max_connections=40 -c max_locks_per_transaction=64 -c checkpoint_timeout=30min - -c maintenance_work_mem=1GB - -c effective_cache_size=16GB - -c work_mem=1GB + -c maintenance_work_mem=4GB + -c effective_cache_size=96GB + -c work_mem=2GB -c max_wal_size=10GB - -c wal_buffers=16MB - -c shared_buffers=8GB - -c max_parallel_workers_per_gather=4 - -c max_parallel_workers=8 - -c max_worker_processes=12 + -c wal_buffers=64MB + -c shared_buffers=32GB + -c max_parallel_workers_per_gather=8 + -c max_parallel_workers=14 + -c max_worker_processes=16 volumes: - ./postgres-data:/var/lib/postgresql/data ports: diff --git a/docker/tuning.md b/docker/tuning.md new file mode 100644 index 0000000..336e61d --- /dev/null +++ b/docker/tuning.md @@ -0,0 +1,68 @@ +# PostgreSQL Tuning + +Tuning notes for the local Docker fwapg instance. Settings in `docker-compose.yml` are tuned for the development machine — adjust for your hardware. + +## Current machine + +- Apple M4 Max Pro +- 128 GB RAM +- 16 cores + +## Settings rationale + +| Setting | Value | Rule of thumb | +|---------|-------|---------------| +| `shared_buffers` | 32GB | ~25% of RAM | +| `effective_cache_size` | 96GB | ~75% of RAM (tells planner about OS cache) | +| `work_mem` | 2GB | Per-operation sort/hash budget. High because fresh queries do large joins and ltree comparisons | +| `maintenance_work_mem` | 4GB | Faster index builds, VACUUM ANALYZE | +| `wal_buffers` | 64MB | Scales with shared_buffers | +| `max_parallel_workers_per_gather` | 8 | Cores per query — aggressive for single-user dev | +| `max_parallel_workers` | 14 | Leave 2 cores for OS/Docker | +| `max_worker_processes` | 16 | Match core count | +| `shm_size` | 36gb | Must exceed shared_buffers (Docker constraint) | + +## Scaling for other machines + +**General formula:** + +``` +shared_buffers = RAM * 0.25 +effective_cache_size = RAM * 0.75 +work_mem = 1-2GB (fresh workload is join-heavy) +maintenance_work_mem = 2-4GB +max_parallel_workers = cores - 2 +max_parallel_workers_per_gather = cores / 2 +max_worker_processes = cores +shm_size = shared_buffers + 4GB +``` + +**Smaller machine (32GB RAM, 8 cores):** + +```yaml +shm_size: 12gb +shared_buffers: 8GB +effective_cache_size: 24GB +work_mem: 1GB +maintenance_work_mem: 2GB +max_parallel_workers_per_gather: 4 +max_parallel_workers: 6 +max_worker_processes: 8 +``` + +## Verifying settings + +```bash +docker compose exec db psql -U postgres -d fwapg -c "SHOW shared_buffers; SHOW work_mem; SHOW effective_cache_size;" +``` + +## Applying changes + +Settings live in `docker-compose.yml` command args. Restart to apply: + +```bash +docker compose down +docker compose up -d db +``` + +Data persists in `postgres-data/` — no reload needed.