From d840006fcdc8d8df8462b17e4d87944f5f4604d2 Mon Sep 17 00:00:00 2001 From: Arnaud Fournier Date: Wed, 29 Apr 2026 10:57:28 +0200 Subject: [PATCH 1/5] Fix(Evaluator): Coalesce null values in arithmetic functions --- coordo-py/coordo/sql/evaluator.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/coordo-py/coordo/sql/evaluator.py b/coordo-py/coordo/sql/evaluator.py index 05c4845..b870601 100644 --- a/coordo-py/coordo/sql/evaluator.py +++ b/coordo-py/coordo/sql/evaluator.py @@ -6,6 +6,7 @@ from pygeofilter.ast import AstType from sqlalchemy import Integer, and_, case, cast, func, or_, select, text +from sqlalchemy.sql.functions import coalesce from coordo.sql.helpers import AGGREGATES, SPATIAL_FUNCTIONS from coordo.sql.mapper import FieldMapper @@ -86,15 +87,15 @@ def evaluate(self, node: AstType, mapper: FieldMapper) -> Any: def arithmetic(self, node, lhs, rhs, *, mapper: FieldMapper): match node.op: case "+": - expr = lhs.expr + rhs.expr + expr = coalesce(lhs.expr, 0) + coalesce(rhs.expr, 0) case "-": - expr = lhs.expr - rhs.expr + expr = coalesce(lhs.expr, 0) - coalesce(rhs.expr, 0) case "/": - expr = lhs.expr / rhs.expr + expr = coalesce(lhs.expr, 0) / coalesce(rhs.expr, 0) case "*": - expr = lhs.expr * rhs.expr + expr = coalesce(lhs.expr, 0) * coalesce(rhs.expr, 0) case "^": - expr = func.pow(lhs.expr, rhs.expr) + expr = func.pow(coalesce(lhs.expr, 0), coalesce(rhs.expr, 1)) case _: raise ValueError("Unsupported operation :", node.op) From 3fe37c8ece9e8cc03e5ca8d9670b0530c3a85187 Mon Sep 17 00:00:00 2001 From: Arnaud Fournier Date: Thu, 30 Apr 2026 22:58:57 +0200 Subject: [PATCH 2/5] Fix coalesce values and update Readme --- README.md | 2 +- coordo-py/coordo/sql/evaluator.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 1183a1a..989a57a 100644 --- a/README.md +++ b/README.md @@ -111,5 +111,5 @@ make catalog Serve a config file ``` -uv run coordo serve data/config.json +uv run coordo serve configs/all4trees_config.json ``` diff --git a/coordo-py/coordo/sql/evaluator.py b/coordo-py/coordo/sql/evaluator.py index b870601..2568184 100644 --- a/coordo-py/coordo/sql/evaluator.py +++ b/coordo-py/coordo/sql/evaluator.py @@ -91,9 +91,9 @@ def arithmetic(self, node, lhs, rhs, *, mapper: FieldMapper): case "-": expr = coalesce(lhs.expr, 0) - coalesce(rhs.expr, 0) case "/": - expr = coalesce(lhs.expr, 0) / coalesce(rhs.expr, 0) + expr = coalesce(lhs.expr, 0) / coalesce(rhs.expr, 1) case "*": - expr = coalesce(lhs.expr, 0) * coalesce(rhs.expr, 0) + expr = coalesce(lhs.expr, 1) * coalesce(rhs.expr, 1) case "^": expr = func.pow(coalesce(lhs.expr, 0), coalesce(rhs.expr, 1)) case _: From 337be638e156e1dc536b35bb4741747fa9708566 Mon Sep 17 00:00:00 2001 From: Arnaud Fournier Date: Thu, 30 Apr 2026 22:59:53 +0200 Subject: [PATCH 3/5] Fix config.json with new evaluator rules --- configs/all4trees_config.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/all4trees_config.json b/configs/all4trees_config.json index a5d6174..d9fae6f 100644 --- a/configs/all4trees_config.json +++ b/configs/all4trees_config.json @@ -43,7 +43,7 @@ "EPF_vertical_distribution": "ind.gini(haut) * 10", "EPF_dominant_height": "ind.avg(haut if haut > percentile(haut, 80))", "EPF_dendro_unique": "ind.list(dmh).flatten().list_unique() / 16 * 10", - "SOIL_structure": "2 * sum(ep1*int(not1) + ep2 * int(not2) + ep3 * int(not3) + ep4 * int(not4) + ep5 * int(not5)) / sum(ep1 + ep2 + ep3 + ep4 + ep5)", + "SOIL_structure": "2 * sum(ep1*int(not1) + ep2 * int(not2) + ep3 * int(not3) + ep4 * int(not4) + ep5 * int(not5)) / (sum(ep1 + ep2 + ep3 + ep4 + ep5) if sum(ep1 + ep2 + ep3 + ep4 + ep5) > 0 else 1)", "SOIL_stability": "(int(slak1) + int(slak2) + int(slak3)) / 3", "SOIL_infiltration_speed": "600 * 300 / ((beer1 + beer2 + beer3 + beer4 + beer5 + beer6 + beer7 + beer8 + beer9 + beer10) * pi() * 30^2) if beer1 != 0 else 0", "SURFACE_species_counts": "list_aggregate(list_concat(list(barba_001.concat_ws('-', tax1_barbA, tax2_barbA, tax3_barbA)), list(barbb_001.concat_ws('-', tax1_barbB, tax2_barbB, tax3_barbB)), list(barbc_001.concat_ws('-', tax1_barbC, tax2_barbC, tax3_barbC))), 'histogram')", From a29ce2f40f2bcb6ff64492448c2ff183706e7ffe Mon Sep 17 00:00:00 2001 From: Arnaud Fournier Date: Wed, 6 May 2026 09:22:57 +0200 Subject: [PATCH 4/5] Remove coalesce in some arithmetic functions and update all4trees config --- configs/all4trees_config.json | 51 ++++++++++++++++++++++--------- coordo-py/coordo/sql/evaluator.py | 4 +-- 2 files changed, 38 insertions(+), 17 deletions(-) diff --git a/configs/all4trees_config.json b/configs/all4trees_config.json index d9fae6f..234bb12 100644 --- a/configs/all4trees_config.json +++ b/configs/all4trees_config.json @@ -31,23 +31,44 @@ "resource": "inventaire_id", "columns": { "geom": "gps.merge().centroid()", + "id": "_id", "for": "for", "cod": "cod", - "double_condition": "ind.list(0 if dhp3 > 0 and dhp4 > 0 else 2)", - "aerial_biomass_volume": "ind.sum(0.0673 * (dens_bois.value * (dhp1 + dhp2 + dhp3 + dhp4 + dhp5 + dhp6 + dhp7 + dhp8 + dhp9 + dhp10)^2 * haut)^0.976 if etat = 1) / (20^2 * pi() / 10000)", - "TREE_species_counts": "ind.histogram(ess_arb)", - "EPF_tree_density": "ind.count(1 if etat = 2) / (20^2 * pi() / 10000)", - "EPF_tree_diversity": "ind.shannon(ess_arb) / 5 * 10", - "EPF_spatial_distribution": "ind.categorical_gini(zon) * 10", - "EPF_diametral_distribution": "ind.gini(dhp1 + dhp2 + dhp3 + dhp4 + dhp5 + dhp6 + dhp7 + dhp8 + dhp9 + dhp10) * 10", - "EPF_vertical_distribution": "ind.gini(haut) * 10", - "EPF_dominant_height": "ind.avg(haut if haut > percentile(haut, 80))", - "EPF_dendro_unique": "ind.list(dmh).flatten().list_unique() / 16 * 10", - "SOIL_structure": "2 * sum(ep1*int(not1) + ep2 * int(not2) + ep3 * int(not3) + ep4 * int(not4) + ep5 * int(not5)) / (sum(ep1 + ep2 + ep3 + ep4 + ep5) if sum(ep1 + ep2 + ep3 + ep4 + ep5) > 0 else 1)", - "SOIL_stability": "(int(slak1) + int(slak2) + int(slak3)) / 3", - "SOIL_infiltration_speed": "600 * 300 / ((beer1 + beer2 + beer3 + beer4 + beer5 + beer6 + beer7 + beer8 + beer9 + beer10) * pi() * 30^2) if beer1 != 0 else 0", - "SURFACE_species_counts": "list_aggregate(list_concat(list(barba_001.concat_ws('-', tax1_barbA, tax2_barbA, tax3_barbA)), list(barbb_001.concat_ws('-', tax1_barbB, tax2_barbB, tax3_barbB)), list(barbc_001.concat_ws('-', tax1_barbC, tax2_barbC, tax3_barbC))), 'histogram')", - "GROUND_species_counts": "list_aggregate(list(tsbf_001.concat_ws('-', tax1_tsbf, tax2_tsbf, tax3_tsbf)), 'histogram')" + "taille_placette": "20^2*pi()/10000", + "projet": "trim('A Kob Ale')", + "biomass_volume": "ind.sum(0.0673 * (dens_bois.value * (dhp1 + dhp2 + dhp3 + dhp4 + dhp5 + dhp6 + dhp7 + dhp8 + dhp9 + dhp10)^2 * haut)^0.976 if etat = 1) / (20^2 * pi() / 10000) / 1000", + "tree_density": "count(1 if ind.etat = 1) / (20^2 * pi() / 10000)", + "tree_pop": "count(1 if ind.etat = 1)", + "richness": "count(unique(ind.ess_arb))", + "relative_abundance": "histogram(ind.ess_arb)", + "epf_tree_density": "count(1 if ind.etat = 1) / (20^2 * pi() / 10000) / 100", + "epf_necromass_pied": "ind.sum(0.0673 * (dens_bois_mort.dens_bois * (dhp1 + dhp2 + dhp3 + dhp4 + dhp5 + dhp6 + dhp7 + dhp8 + dhp9 + dhp10)^2 * haut)^0.976 if etat = 2)", + "epf_necromass_sol": "ind.sum(3.1415926535 / 3 * ((dhp_b / 2)^2 + ((dhp_e / 2)^2 + dhp_b / 2 + dhp_e / 2) * longueur * dens_bois_mort.dens_bois) if etat = 3)", + "epf_deadWood": "((ind.sum(0.0673 * (dens_bois_mort.dens_bois * (dhp1 + dhp2 + dhp3 + dhp4 + dhp5 + dhp6 + dhp7 + dhp8 + dhp9 + dhp10)^2 * haut)^0.976 if etat = 2) + ind.sum(pi() / 3 * ((dhp_b / 2)^2 + ((dhp_e / 2)^2 + dhp_b / 2 + dhp_e / 2) * longueur * dens_bois_mort.dens_bois) if etat = 3)) / ind.sum(0.0673 * (dens_bois.value * (dhp1 + dhp2 + dhp3 + dhp4 + dhp5 + dhp6 + dhp7 + dhp8 + dhp9 + dhp10)^2 * haut)^0.976 if etat = 1)) * 10", + "epf_tree_diversity": "shannon(ind.ess_arb) / 5 * 10", + "epf_spatial_distribution": "ind.categorical_gini(zon) * 10", + "epf_diameter_distribution": "ind.gini(dhp1 + dhp2 + dhp3 + dhp4 + dhp5 + dhp6 + dhp7 + dhp8 + dhp9 + dhp10) * 10", + "epf_vertical_distribution": "gini(ind.haut) * 10", + "epf_dominant_height": "ind.avg(haut if haut > percentile(haut, 80)) * 10 / 40", + "epf_microhabitats": "ind.list(dmh).flatten().list_unique() * 10 / 16", + "soil_structure": "sum(ep1*int(not1) + ep2*int(not2) + ep3*int(not3) + ep4*int(not4) + ep5*int(not5)) / (sum(ep1 + ep2 + ep3 + ep4 + ep5) if sum(ep1 + ep2 + ep3 + ep4 + ep5) > 0 else 1) * 2", + "soil_composition": "count(1)", + "ero_rainfall_and_wind": "meteo.concat_ws('-', pluv, vent)", + "ero_couv_slope_and_cover": "inventaire_external.concat_ws('-', pent, couv)", + "ero_soil_stability": "(int(slak1) + int(slak2) + int(slak3)) / 3", + "ero_water_seepage": "600 * 300 / ((beer1 + beer2 + beer3 + beer4 + beer5 + beer6 + beer7 + beer8 + beer9 + beer10) * pi() * 30^2) if beer1 != 0 else 0", + "soil_fauna_density": "sum(tsbf_001.nomb_tsbf) / (20^2 * pi() / 10000)", + "soil_fauna_diversity": "list_unique(list(tsbf_001.concat_ws('-', tax1_tsbf, tax2_tsbf, tax3_tsbf)))", + "soil_fauna_abundance_tax1": "histogram(tsbf_001.tax1_tsbf)", + "soil_fauna_abundance_tax2": "histogram(tsbf_001.tax2_tsbf)", + "soil_fauna_abundance_tax3": "histogram(tsbf_001.tax3_tsbf)", + "surface_fauna_density": "sum(barba_001.nomb_barbA + barbb_001.nomb_barbB + barbc_001.nomb_barbC + barbd_001.nomb_barbD) / (20^2 * pi() / 10000)", + "surface_fauna_diversity": "list_unique(list_concat(list(barba_001.concat_ws('-', tax1_barbA, tax2_barbA, tax3_barbA)), list(barbb_001.concat_ws('-', tax1_barbB, tax2_barbB, tax3_barbB)), list(barbc_001.concat_ws('-', tax1_barbC, tax2_barbC, tax3_barbC)), list(barbd_001.concat_ws('-', tax1_barbD, tax2_barbD, tax3_barbD))))", + "surface_fauna_abundance_tax1": "list_aggregate(list_concat(list(barba_001.tax1_barbA), list(barbb_001.tax1_barbB), list(barbc_001.tax1_barbC), list(barbd_001.tax1_barbD)), 'histogram')", + "surface_fauna_abundance_tax2": "list_aggregate(list_concat(list(barba_001.tax2_barbA), list(barbb_001.tax2_barbB), list(barbc_001.tax2_barbC), list(barbd_001.tax2_barbD)), 'histogram')", + "surface_fauna_abundance_tax3": "list_aggregate(list_concat(list(barba_001.tax3_barbA), list(barbb_001.tax3_barbB), list(barbc_001.tax3_barbC), list(barbd_001.tax3_barbD)), 'histogram')", + "surface_fauna_abundance": "list_aggregate(list_concat(list(barba_001.concat_ws('-', tax1_barbA, tax2_barbA, tax3_barbA)), list(barbb_001.concat_ws('-', tax1_barbB, tax2_barbB, tax3_barbB)), list(barbc_001.concat_ws('-', tax1_barbC, tax2_barbC, tax3_barbC)), list(barbd_001.concat_ws('-', tax1_barbD, tax2_barbD, tax3_barbD))), 'histogram')", + "soil_fauna_abundance": "list_aggregate(list(tsbf_001.concat_ws('-', tax1_tsbf, tax2_tsbf, tax3_tsbf)), 'histogram')" }, "popup": { "trigger": "click" diff --git a/coordo-py/coordo/sql/evaluator.py b/coordo-py/coordo/sql/evaluator.py index 2568184..84e0e5e 100644 --- a/coordo-py/coordo/sql/evaluator.py +++ b/coordo-py/coordo/sql/evaluator.py @@ -91,11 +91,11 @@ def arithmetic(self, node, lhs, rhs, *, mapper: FieldMapper): case "-": expr = coalesce(lhs.expr, 0) - coalesce(rhs.expr, 0) case "/": - expr = coalesce(lhs.expr, 0) / coalesce(rhs.expr, 1) + expr = lhs.expr / rhs.expr case "*": expr = coalesce(lhs.expr, 1) * coalesce(rhs.expr, 1) case "^": - expr = func.pow(coalesce(lhs.expr, 0), coalesce(rhs.expr, 1)) + expr = func.pow(lhs.expr, rhs.expr) case _: raise ValueError("Unsupported operation :", node.op) From 1577c96826d30b498a50038954924e21a55b7bd3 Mon Sep 17 00:00:00 2001 From: Arnaud Fournier Date: Thu, 7 May 2026 08:01:14 +0200 Subject: [PATCH 5/5] Remove product coalesce --- configs/all4trees_config.json | 2 +- coordo-py/coordo/sql/evaluator.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/configs/all4trees_config.json b/configs/all4trees_config.json index 234bb12..7050441 100644 --- a/configs/all4trees_config.json +++ b/configs/all4trees_config.json @@ -51,7 +51,7 @@ "epf_vertical_distribution": "gini(ind.haut) * 10", "epf_dominant_height": "ind.avg(haut if haut > percentile(haut, 80)) * 10 / 40", "epf_microhabitats": "ind.list(dmh).flatten().list_unique() * 10 / 16", - "soil_structure": "sum(ep1*int(not1) + ep2*int(not2) + ep3*int(not3) + ep4*int(not4) + ep5*int(not5)) / (sum(ep1 + ep2 + ep3 + ep4 + ep5) if sum(ep1 + ep2 + ep3 + ep4 + ep5) > 0 else 1) * 2", + "soil_structure": "2 * sum(ep1*int(not1) + ep2*int(not2) + ep3*int(not3) + ep4*int(not4) + ep5*int(not5)) / sum(ep1 + ep2 + ep3 + ep4 + ep5)", "soil_composition": "count(1)", "ero_rainfall_and_wind": "meteo.concat_ws('-', pluv, vent)", "ero_couv_slope_and_cover": "inventaire_external.concat_ws('-', pent, couv)", diff --git a/coordo-py/coordo/sql/evaluator.py b/coordo-py/coordo/sql/evaluator.py index 84e0e5e..fd758a6 100644 --- a/coordo-py/coordo/sql/evaluator.py +++ b/coordo-py/coordo/sql/evaluator.py @@ -93,7 +93,7 @@ def arithmetic(self, node, lhs, rhs, *, mapper: FieldMapper): case "/": expr = lhs.expr / rhs.expr case "*": - expr = coalesce(lhs.expr, 1) * coalesce(rhs.expr, 1) + expr = lhs.expr * rhs.expr case "^": expr = func.pow(lhs.expr, rhs.expr) case _: