From 9968576c6711525745a2a71bb2914380439bf404 Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Wed, 29 Jul 2026 23:42:33 -0400 Subject: [PATCH 01/98] Make the DNASeq injector dormant while dnaseq processing is redone IsolatesHTS no longer describes how DNASeq datasets are processed on this branch, and both of its methods fail on the new shape: - injectTemplates() derived per-sample gbrowse/jbrowse databases from getSampleList(), which keys samples on organismAbbrev + datasetClassCategory + experimentName. The new dnaseqExperiment class carries an empty category while its samples span two ("Genetic variation" for SNPs, "Structural variation" for CNVs), so the key cannot match and getSampleList() throws "No sample names found for dataset ...". - addModelReferences() registered SnpQuestions / SnpRecordClasses references. That XML is not in the compiled model and is to be superseded by the variation record. Both bodies are commented out rather than deleted, and the class stays in place as a no-op so the presenters naming it keep resolving. getPropertiesDeclaration() is left alone: presenters still supply hasCNVData, and DatasetInjector.addPropValues validates supplied props against the declaration. Note the deeper issue for whoever writes the replacement: keying the experiment -> sample lookup on a single datasetClassCategory cannot work for an experiment that aggregates SNP and CNV samples, even once dnaseqExperiment declares a category. Co-Authored-By: Claude Opus 5 --- .../model/datasetInjector/IsolatesHTS.java | 24 ++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/Model/src/main/java/org/apidb/apicommon/model/datasetInjector/IsolatesHTS.java b/Model/src/main/java/org/apidb/apicommon/model/datasetInjector/IsolatesHTS.java index ccec9aff87..a925988aba 100644 --- a/Model/src/main/java/org/apidb/apicommon/model/datasetInjector/IsolatesHTS.java +++ b/Model/src/main/java/org/apidb/apicommon/model/datasetInjector/IsolatesHTS.java @@ -3,10 +3,30 @@ import org.apidb.apicommon.datasetPresenter.DatasetInjector; import java.util.List; +/** + * DNASeq dataset injector. + * + * Both method bodies are commented out for the dnaseq-merge-experiments work: DNASeq + * datasets are processed differently now, so neither what this injected nor what it + * referenced describes them any more. Concretely: + * + * - injectTemplates() derived per-sample gbrowse/jbrowse databases from getSampleList(), + * which keys samples on organismAbbrev + datasetClassCategory + experimentName. The new + * dnaseqExperiment class carries an empty category while its samples span two + * ("Genetic variation" for SNPs, "Structural variation" for CNVs), so that key cannot + * match and getSampleList() throws. + * - addModelReferences() registered SnpQuestions / SnpRecordClasses references. That XML + * is not in the compiled model and is to be superseded by the variation record. + * + * The class stays in place as a no-op so presenters may keep naming it while the new + * processing lands; getPropertiesDeclaration() is left alone so the hasCNVData prop that + * presenters still supply remains declared. + */ public class IsolatesHTS extends DatasetInjector { @Override public void injectTemplates() { + /* commented out for dnaseq-merge-experiments; see the class comment above String datasetName = getDatasetName(); setOrganismAbbrevFromDatasetName(); @@ -65,10 +85,12 @@ public void injectTemplates() { } //System.err.println("short attribution" + getPropValue("shortAttribution")); + */ } @Override public void addModelReferences() { + /* commented out for dnaseq-merge-experiments; see the class comment above // NGS SNPs addWdkReference("SnpRecordClasses.SnpRecordClass", "question", "SnpQuestions.NgsSnpBySourceId"); addWdkReference("SnpRecordClasses.SnpRecordClass", "question", "SnpQuestions.NgsSnpsByIsolateGroup"); @@ -107,7 +129,7 @@ public void addModelReferences() { addWdkReference("SampleRecordClasses.SampleRecordClass", "table", "Datasets"); addWdkReference("SampleRecordClasses.SampleRecordClass", "table", "Characteristics"); addWdkReference("SampleRecordClasses.SampleRecordClass", "table", "ProcessedSample"); - + */ } From 60a3e36ea37c66c6df4c612205883334b3e37994 Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Wed, 29 Jul 2026 23:42:48 -0400 Subject: [PATCH 02/98] Stop consuming communitycount, which this branch stopped producing Commit 643ddb442 commented out the CommunityCount intermediate table and the communityCount column of OrganismAttributes in apiTuningManager.xml (it read from a hardcoded usercomments_userdb_betan.mappedComment), but left every consumer in place, so model load failed with: Database error while attempting to parse sqlQuery OrganismAttributes.organismAttrs: ERROR: column oa.communitycount does not exist Consumers brought into line with the producer, following the snpcount/popsetcount precedent already in these files: - organismAttributeQueries.xml: the communitycount column declaration and its select expression are commented out. - organismRecords.xml: the communitycount columnAttribute is commented out. - organismQuestions.xml: communitycount removed from five summary attribute lists, which is mandatory since WDK resolves every attribute a summary names. community_flag is kept, as a literal 'none' rather than derived from the missing column. That is exactly what "CASE oa.communitycount WHEN 0 THEN 'none'" produced for a zero count, so the organism page renders as it would with no comments, rather than losing an attribute that organismRecords.xml:305 and its red-dot still use. Deriving it from NULL would have been worse than removing it: CASE NULL WHEN 0 falls through to ELSE, so every organism would have advertised comments it does not have. The individuals.txt ontology entry is left alone, matching how snpcount and popsetcount were handled: the category tree tolerates entries for absent attributes. Co-Authored-By: Claude Opus 5 --- Model/lib/wdk/model/questions/organismQuestions.xml | 10 +++++----- .../lib/wdk/model/records/organismAttributeQueries.xml | 8 +++++--- Model/lib/wdk/model/records/organismRecords.xml | 6 ++++-- 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/Model/lib/wdk/model/questions/organismQuestions.xml b/Model/lib/wdk/model/questions/organismQuestions.xml index 1002fed34d..ec612c42ee 100644 --- a/Model/lib/wdk/model/questions/organismQuestions.xml +++ b/Model/lib/wdk/model/questions/organismQuestions.xml @@ -22,17 +22,17 @@ @@ -69,12 +69,12 @@ This search allows you to identify organisms based on the name of the organism. recordClassRef="OrganismRecordClasses.OrganismRecordClass"> diff --git a/Model/lib/wdk/model/records/organismAttributeQueries.xml b/Model/lib/wdk/model/records/organismAttributeQueries.xml index 31da7de609..248411ee50 100644 --- a/Model/lib/wdk/model/records/organismAttributeQueries.xml +++ b/Model/lib/wdk/model/records/organismAttributeQueries.xml @@ -101,7 +101,7 @@ - + @@ -158,7 +158,7 @@ nullif(oa.gocount,0) as gocount, nullif(oa.tfbscount,0) as tfbscount, nullif(oa.proteomicscount,0) as proteomicscount, - nullif(oa.communitycount,0) as communitycount, + --nullif(oa.communitycount,0) as communitycount, CASE oa.isOrganellar WHEN 0 THEN 'no' ELSE 'yes' END as isOrganellar_flag, CASE oa.arraygenecount WHEN 0 THEN 'none' ELSE 'yes' END as array_flag, CASE oa.chipchipgenecount WHEN 0 THEN 'none' ELSE 'yes' END as chipchip_flag, @@ -168,7 +168,9 @@ CASE oa.ecnumbercount WHEN 0 THEN 'none' ELSE 'yes' END as ec_flag, CASE oa.gocount WHEN 0 THEN 'none' ELSE 'yes' END as go_flag, CASE oa.rtpcrcount WHEN 0 THEN 'none' ELSE 'yes' END as rtpcr_flag, - CASE oa.communitycount WHEN 0 THEN 'none' ELSE 'yes' END as community_flag, + -- was: CASE oa.communitycount WHEN 0 THEN 'none' ELSE 'yes' END; communityCount is + -- disabled in apiTuningManager.xml, and 'none' is what it yielded for a zero count + 'none' as community_flag, CASE oa.tfbscount WHEN 0 THEN 'none' ELSE 'yes' END as tfbs_flag, CASE oa.hasHTSIsolate WHEN 0 THEN 'none' ELSE 'yes' END as hasHTSIsolate_flag, --CASE oa.hasPopset WHEN 0 THEN 'none' ELSE 'yes' END as hasPopset_flag, diff --git a/Model/lib/wdk/model/records/organismRecords.xml b/Model/lib/wdk/model/records/organismRecords.xml index a1bb762fd9..3b9485ca19 100644 --- a/Model/lib/wdk/model/records/organismRecords.xml +++ b/Model/lib/wdk/model/records/organismRecords.xml @@ -180,8 +180,10 @@ help="Number of genes that have evidence of protein expression based on Mass Spectrometry studies" align="right"/> - + From 28b06e44839edea3786d693caa901fcf815be670 Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Wed, 29 Jul 2026 23:43:17 -0400 Subject: [PATCH 03/98] Inject the DeRisi time series questions instead of hardcoding them These two questions were written by hand in geneQuestions.xml while the attribute their summaries name is generated by the presenter: summary="...,exprGraphAttrpfal3D7_microarrayExpression_Derisi_TimeSeries_RSRC_expr_graph" Expression.java:136 builds that name and injects the textAttribute that defines it. So a hand-written question depended on generated output, coupled only by a name string with nothing enforcing that both exist. On an instance where this dataset is not loaded, the presenter is skipped and the question survives, referencing an attribute nobody generated: Summary attribute field [exprGraphAttrpfal3D7_..._expr_graph] defined in question [GeneQuestions.GenesByMicroarraypfal3D7_..._RSRC] is invalid The questions are curated rather than generic -- they use the PFTimeSeries vocabulary queries, an extra samples_fc_comp_generic param, and fold_change_chosen_display in place of fold_change_chosen -- which is why the presenter sets hasMultipleSamples, hasPercentileData and hasPageData to false and why they were hardcoded in the first place. So they move to their own microarrayDeRisiTimeSeries.dst rather than being folded into the generic expression templates, and MicroarrayDeRisiTimeSeries injects them. Question and attribute now appear and disappear together. Only three values are parameterised -- ${datasetName} for the question and graph attribute names, ${includeProjects} and ${includeProjectsExcludeEuPathDB} for scoping, both already set by Expression.java to precisely the literals that were hardcoded. Display strings stay literal because they describe this experiment. addModelReferences now derives the question names from getDatasetName() rather than spelling out the dataset, so the references cannot drift from what is injected. Not yet verified: that the generated question is equivalent to the deleted static XML. Nothing is generated on an instance where this dataset is absent, so this branch can only show that its absence is handled. Before this reaches a full-data site, run a build with the dataset declared and diff the generated question against the static version in this commit's parent. Co-Authored-By: Claude Opus 5 --- Model/lib/dst/microarrayDeRisiTimeSeries.dst | 193 ++++++++++++++++++ .../lib/wdk/model/questions/geneQuestions.xml | 181 +--------------- .../PlasmoDB/MicroarrayDeRisiTimeSeries.java | 22 +- 3 files changed, 215 insertions(+), 181 deletions(-) create mode 100644 Model/lib/dst/microarrayDeRisiTimeSeries.dst diff --git a/Model/lib/dst/microarrayDeRisiTimeSeries.dst b/Model/lib/dst/microarrayDeRisiTimeSeries.dst new file mode 100644 index 0000000000..598f48f3a5 --- /dev/null +++ b/Model/lib/dst/microarrayDeRisiTimeSeries.dst @@ -0,0 +1,193 @@ +[templateStart] +name=microarrayDeRisiTimeSeriesFoldChangeQuestion +anchorFile=ApiCommonModel/Model/lib/wdk/model/questions/geneQuestions.xml +prop=datasetName +prop=includeProjects +prop=includeProjectsExcludeEuPathDB +>templateTextStart< + + + + + + + + + + 1 + + + + + + + + + +
+After selecting samples you have the option to take the average, minimum, or maximum expression value within each group. (If choosing only one sample from a group, the selected 'operation' will not affect your results). Time series experiments will offer an extra parameter called "Global min/max" which allows you to filter your results further. Finally, you can choose the directionality and the magnitude of the difference. For example, selecting up-regulated with a fold difference of 2 will only show results where the comparator is twice that of the reference. +

+ + + ]]> + + + + + + Display the histogram of the values of this attribute + int + + + + + Display the histogram of the values of this attribute + int + + + + + + Display the histogram of the values of this attribute + int + + + + + Display the histogram of the values of this attribute + int + + + + + Display the histogram of the values of this attribute + int + + + + + Display the histogram of the values of this attribute + int + + + + + Display the histogram of the values of this attribute + int + + + + + Display the histogram of the values of this attribute + int + + + + + Display the histogram of the values of this attribute + int + + + + + Display the histogram of the values of this attribute + int + + + + + + fold_change + + +>templateTextEnd< + +[templateStart] +name=microarrayDeRisiTimeSeriesPercentileQuestion +anchorFile=ApiCommonModel/Model/lib/wdk/model/questions/geneQuestions.xml +prop=datasetName +prop=includeProjects +prop=includeProjectsExcludeEuPathDB +>templateTextStart< + + + + + + + + + + + + + + + + +
+ + + ]]> + + + + + + Display the histogram of the values of this attribute + int + + + + + Display the histogram of the values of this attribute + int + + + + + + percentile + + +>templateTextEnd< diff --git a/Model/lib/wdk/model/questions/geneQuestions.xml b/Model/lib/wdk/model/questions/geneQuestions.xml index 126166f30b..4515f884b1 100644 --- a/Model/lib/wdk/model/questions/geneQuestions.xml +++ b/Model/lib/wdk/model/questions/geneQuestions.xml @@ -3696,182 +3696,11 @@ For further experiment details please refer the data sources listed below. ]]> - - - - - - - - - 1 - - - - - - - - - -
-After selecting samples you have the option to take the average, minimum, or maximum expression value within each group. (If choosing only one sample from a group, the selected 'operation' will not affect your results). Time series experiments will offer an extra parameter called "Global min/max" which allows you to filter your results further. Finally, you can choose the directionality and the magnitude of the difference. For example, selecting up-regulated with a fold difference of 2 will only show results where the comparator is twice that of the reference. -

- - - ]]> - - - - - - Display the histogram of the values of this attribute - int - - - - - Display the histogram of the values of this attribute - int - - - - - - Display the histogram of the values of this attribute - int - - - - - Display the histogram of the values of this attribute - int - - - - - Display the histogram of the values of this attribute - int - - - - - Display the histogram of the values of this attribute - int - - - - - Display the histogram of the values of this attribute - int - - - - - Display the histogram of the values of this attribute - int - - - - - Display the histogram of the values of this attribute - int - - - - - Display the histogram of the values of this attribute - int - - - - - - fold_change - - - - - - - - - - - - - - - - - - - -
- - - ]]> - - - - - - Display the histogram of the values of this attribute - int - - - - - Display the histogram of the values of this attribute - int - - - - - - percentile - - + + + diff --git a/Model/src/main/java/org/apidb/apicommon/model/datasetInjector/custom/PlasmoDB/MicroarrayDeRisiTimeSeries.java b/Model/src/main/java/org/apidb/apicommon/model/datasetInjector/custom/PlasmoDB/MicroarrayDeRisiTimeSeries.java index 55370b7bb7..4bf89b4b76 100644 --- a/Model/src/main/java/org/apidb/apicommon/model/datasetInjector/custom/PlasmoDB/MicroarrayDeRisiTimeSeries.java +++ b/Model/src/main/java/org/apidb/apicommon/model/datasetInjector/custom/PlasmoDB/MicroarrayDeRisiTimeSeries.java @@ -8,13 +8,22 @@ public class MicroarrayDeRisiTimeSeries extends MicroarrayTwoChannelReferenceDes public void injectTemplates() { super.injectTemplates(); - // Questions are hard coded in the model w/ the same name that would have been injected - - // we are setting hasPercentile to false so must inject these + // we are setting hasPercentile to false so must inject these setPropValue("graphTextAttrName", "pctGraphAttr" + getDatasetName() + "_pct_graph"); injectTemplate("expressionGraphAttributesPercentile"); injectTemplate("graphTextAttributeCategory"); + // These two questions are curated rather than generic — they use the PFTimeSeries + // vocabulary queries and an extra comparison-samples param — so hasMultipleSamples / + // hasPercentileData / hasPageData stay false and the questions come from + // microarrayDeRisiTimeSeries.dst instead of the generic expression templates. + // + // They used to be hardcoded in geneQuestions.xml, which meant they existed even on an + // instance where this dataset is not loaded, referencing expr/pct graph attributes that + // only exist when this presenter runs. Injecting them makes question and attribute + // appear and disappear together, so the model stays dataset driven. + injectTemplate("microarrayDeRisiTimeSeriesFoldChangeQuestion"); + injectTemplate("microarrayDeRisiTimeSeriesPercentileQuestion"); } @Override @@ -22,8 +31,11 @@ public void addModelReferences() { super.addModelReferences(); addWdkReference("TranscriptRecordClasses.TranscriptRecordClass", "question", "GeneQuestions.GenesByProfileSimilarity"); - addWdkReference("TranscriptRecordClasses.TranscriptRecordClass", "question", "GeneQuestions.GenesByMicroarraypfal3D7_microarrayExpression_Derisi_TimeSeries_RSRC"); - addWdkReference("TranscriptRecordClasses.TranscriptRecordClass", "question", "GeneQuestions.GenesByMicroarraypfal3D7_microarrayExpression_Derisi_TimeSeries_RSRCPercentile"); + // derived from the dataset rather than spelled out, so these track the injected + // question names above instead of drifting from them + String questionName = "GeneQuestions.GenesByMicroarray" + getDatasetName(); + addWdkReference("TranscriptRecordClasses.TranscriptRecordClass", "question", questionName); + addWdkReference("TranscriptRecordClasses.TranscriptRecordClass", "question", questionName + "Percentile"); } From e78539028ad041223af169726cbaead1dcf51374 Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Wed, 29 Jul 2026 23:43:36 -0400 Subject: [PATCH 04/98] Drop a dataset-specific attribute from a generic compound search CompoundQuestions.CompoundsByFoldChange named one dataset's graph column in its summary: summary="...,metaboliteProfiles_LlinasMetabolites_RSRC_metabolite_graph" That attribute is injected per metabolomics dataset by metabolomics.dst, so naming a particular dataset's copy in a generic search has two consequences: model load fails wherever that dataset is not loaded ("Summary attribute field [...] is invalid"), and where it is loaded the search shows that one dataset's graph regardless of what the user searched for. Co-Authored-By: Claude Opus 5 --- Model/lib/wdk/model/questions/compoundQuestions.xml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/Model/lib/wdk/model/questions/compoundQuestions.xml b/Model/lib/wdk/model/questions/compoundQuestions.xml index e3a8ba1820..68b172b35c 100644 --- a/Model/lib/wdk/model/questions/compoundQuestions.xml +++ b/Model/lib/wdk/model/questions/compoundQuestions.xml @@ -273,10 +273,15 @@ Find compounds by metabolite levels - + + />
From 9ff609477514bcddbb9483d623a37d63016c416e Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Thu, 30 Jul 2026 01:03:18 -0400 Subject: [PATCH 05/98] Add VariationAttributes tuning table One row per variant locus (4,390,908 in unidb_shu_a), backing a new "variation" WDK record that replaces the deprecated snp record. Holds only columns that are derived, aggregated, or require a join. Intrinsic per-locus facts -- variant_type, is_coding, all 22 snp_*/indel_* allele columns, the strain counts -- are read directly from apidb.VariationFeature, which is already one row per locus with a unique source_id. That split keeps ~570MB of derived strings out of the tuning table and gives every column one known source. Both internalDependency elements are load-bearing: built before TranscriptAttributes the gene aggregate is silently empty rather than an error, and built before GenomicSeqAttributes the project_id/organism join drops every row. Notable derivations: - gene_ids is a string_agg, not a lookup: 25,545 loci overlap more than one gene, so the multi-gene case is correct by construction. - most_severe_impact and effect_summary are split per caller, because snpeff and product_call disagree on 19% of paired calls and that disagreement is content rather than noise. - collapsed_allele renders both classes for a MIXED locus (e.g. "A>C; A>AC"), which a single major/minor pair would have destroyed. The SELECT was verified against unidb_shu_a before being committed: 4,390,908 rows, 100% join coverage against GenomicSeqAttributes, and the expected values at three loci covering SNV, INDEL, and MIXED. Design: docs/superpowers/specs/2026-07-30-variation-record-design.md and plans/2026-07-30-variation-record.md in agentic-veupath-dev. Co-Authored-By: Claude Opus 5 --- .../xml/tuningManager/apiTuningManager.xml | 103 ++++++++++++++++++ 1 file changed, 103 insertions(+) diff --git a/Model/lib/xml/tuningManager/apiTuningManager.xml b/Model/lib/xml/tuningManager/apiTuningManager.xml index 52c0d7c5ec..33e2d90728 100644 --- a/Model/lib/xml/tuningManager/apiTuningManager.xml +++ b/Model/lib/xml/tuningManager/apiTuningManager.xml @@ -1086,6 +1086,109 @@ + + + One row per variant locus, for the "variation" WDK record. Holds ONLY columns that + are derived, aggregated, or require a join; intrinsic per-locus facts are read + directly from apidb.VariationFeature by the WDK attribute query, since that table + is already one row per locus with a unique source_id. + + Both internalDependency elements are load-bearing: built before + TranscriptAttributes the gene aggregate is silently empty rather than an error, and + built before GenomicSeqAttributes the project_id/organism join drops every row. + + See docs/superpowers/specs/2026-07-30-variation-record-design.md in + agentic-veupath-dev. + + + + + + + + + + ', v.snp_minor_allele) END, + CASE WHEN v.indel_minor_allele IS NOT NULL THEN concat(v.indel_ref_allele, '>', v.indel_minor_allele) END + ) AS collapsed_allele, + CASE WHEN v.snp_minor_allele_frequency IS NULL + AND v.indel_minor_allele_frequency IS NULL THEN NULL + ELSE greatest(coalesce(v.snp_minor_allele_frequency, 0), + coalesce(v.indel_minor_allele_frequency, 0)) END AS collapsed_minor_allele_frequency + FROM apidb.VariationFeature v + JOIN GenomicSeqAttributes g ON g.source_id = v.sequence_source_id + JOIN sres.ExternalDatabaseRelease r ON r.external_database_release_id = v.external_database_release_id + JOIN sres.ExternalDatabase d ON d.external_database_id = r.external_database_id + LEFT JOIN DatasetPresenter dp ON dp.name = d.name + LEFT JOIN gene_agg ga ON ga.sequence_source_id = v.sequence_source_id AND ga.location = v.location + LEFT JOIN effect_agg ea ON ea.sequence_source_id = v.sequence_source_id AND ea.location = v.location + ]]> + + + + + + + + + + + + From 9bcf147595561228af0fd3ebb36352e4d5e4827a Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Thu, 30 Jul 2026 01:17:55 -0400 Subject: [PATCH 06/98] Add a minimal buildable variation record Record class, alias query, and the tuning-backed identity/location attributes. Builds green and loads by ID; 13 attributes registered. sequence_source_id and location are exposed as first-class non-internal attributes rather than being parsed back out of source_id, because they are the coordinate a future VCF/tabix lookup and an EDA sample join both key on. Two things differ from the written plan, both discovered by building it: - The snp imports in apiCommonModel.xml sit inside an "UNCOMMENT WHEN SNPS are AVAILABLE" comment block, so inserting after them would have silently disabled the variation record. Imports are placed after that block instead. - variationTableQueries.xml is not created yet. WDK's RELAX NG schema requires a querySet to contain at least one query, so the intended empty-but-valid placeholder fails validation. The file arrives with its first real query. Temporarily reads jbrestel.VariationAttributes; see Task 14 of the plan. Co-Authored-By: Claude Opus 5 --- Model/lib/wdk/apiCommonModel.xml | 4 + .../records/variationAttributeQueries.xml | 80 ++++++++++++++ .../wdk/model/records/variationRecords.xml | 104 ++++++++++++++++++ 3 files changed, 188 insertions(+) create mode 100644 Model/lib/wdk/model/records/variationAttributeQueries.xml create mode 100644 Model/lib/wdk/model/records/variationRecords.xml diff --git a/Model/lib/wdk/apiCommonModel.xml b/Model/lib/wdk/apiCommonModel.xml index 800b75e9a5..afb78793b8 100644 --- a/Model/lib/wdk/apiCommonModel.xml +++ b/Model/lib/wdk/apiCommonModel.xml @@ -410,6 +410,10 @@ --> + + + + + + + + + Variant_Pf3D7_01_v3_100057 + PlasmoDB + + + + Variant_11L3_v3_26886 + TriTrypDB + + + + Variant_Chr1_A_fumigatus_Af293_1000005 + FungiDB + + + + SELECT count(*) FROM jbrestel.VariationAttributes + + + + + + + + + + + + + + + + + + + + + + + + + + + ', SUBSTR(va.organism, 1, 1), '.', + REGEXP_REPLACE(SUBSTR(va.organism, strpos(va.organism, ' ')), + '[[:space:]]+', CONCAT(chr(38), 'nbsp;')), + '') AS formatted_organism, + va.ncbi_tax_id, + va.dataset + FROM jbrestel.VariationAttributes va + ]]> + + + + + diff --git a/Model/lib/wdk/model/records/variationRecords.xml b/Model/lib/wdk/model/records/variationRecords.xml new file mode 100644 index 0000000000..d1d8d5fa5b --- /dev/null +++ b/Model/lib/wdk/model/records/variationRecords.xml @@ -0,0 +1,104 @@ + + + + + + + + Variant_Pf3D7_01_v3_100057 + PlasmoDB + + + + Variant_Pf3D7_01_v3_100057 + + + + Variant_11L3_v3_26886 + TriTrypDB + + + + Variant_Chr1_A_fumigatus_Af293_1000005 + FungiDB + + + + source_id + project_id + + + + + + + + + + + + + + 500 + + + + 1000000 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $$organism_text$$ ]]> + + + + + + + + + + From 48f13bc81520054bea77946abdf4af918bc8e371 Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Thu, 30 Jul 2026 01:25:15 -0400 Subject: [PATCH 07/98] Add variation classification attributes from VariationFeature First attribute query reading apidb.VariationFeature directly, per the sourcing rule: intrinsic per-locus facts come from the base table, only derived/joined columns come from the tuning table. --- .../records/variationAttributeQueries.xml | 22 +++++++++++++++++++ .../wdk/model/records/variationRecords.xml | 8 +++++++ 2 files changed, 30 insertions(+) diff --git a/Model/lib/wdk/model/records/variationAttributeQueries.xml b/Model/lib/wdk/model/records/variationAttributeQueries.xml index 22f6236576..a0ef6fbe27 100644 --- a/Model/lib/wdk/model/records/variationAttributeQueries.xml +++ b/Model/lib/wdk/model/records/variationAttributeQueries.xml @@ -76,5 +76,27 @@ + + + + + + + + + + + + diff --git a/Model/lib/wdk/model/records/variationRecords.xml b/Model/lib/wdk/model/records/variationRecords.xml index d1d8d5fa5b..859c39fc1d 100644 --- a/Model/lib/wdk/model/records/variationRecords.xml +++ b/Model/lib/wdk/model/records/variationRecords.xml @@ -88,6 +88,14 @@ + + + + + + $$organism_text$$ ]]> From d35e5570e76125c8f107f3f9b5bdc3f526796582 Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Thu, 30 Jul 2026 01:30:43 -0400 Subject: [PATCH 08/98] Add SNP and Indel allele attributes as separate sections Both allele classes are preserved rather than collapsed: 129,850 MIXED loci populate snp_* and indel_* simultaneously and describe the same strains two ways, so a single major/minor pair would silently lose half the data. HGVS attributes carry help text explaining that the major-allele HGVS is empty whenever the major allele equals the reference. --- .../records/variationAttributeQueries.xml | 32 ++++++++++++++- .../wdk/model/records/variationRecords.xml | 39 +++++++++++++++++++ 2 files changed, 70 insertions(+), 1 deletion(-) diff --git a/Model/lib/wdk/model/records/variationAttributeQueries.xml b/Model/lib/wdk/model/records/variationAttributeQueries.xml index a0ef6fbe27..c74e0a4a88 100644 --- a/Model/lib/wdk/model/records/variationAttributeQueries.xml +++ b/Model/lib/wdk/model/records/variationAttributeQueries.xml @@ -85,12 +85,42 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Date: Thu, 30 Jul 2026 01:37:04 -0400 Subject: [PATCH 09/98] Add variation strain and call statistic attributes Grouped together deliberately: these aggregates are what the future per-strain VCF-backed table will detail, so it lands beside them. --- .../wdk/model/records/variationAttributeQueries.xml | 13 ++++++++++++- Model/lib/wdk/model/records/variationRecords.xml | 13 +++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/Model/lib/wdk/model/records/variationAttributeQueries.xml b/Model/lib/wdk/model/records/variationAttributeQueries.xml index c74e0a4a88..593ac64542 100644 --- a/Model/lib/wdk/model/records/variationAttributeQueries.xml +++ b/Model/lib/wdk/model/records/variationAttributeQueries.xml @@ -104,6 +104,13 @@ + + + + + + + + + + + + + + + + Date: Thu, 30 Jul 2026 01:42:57 -0400 Subject: [PATCH 10/98] Add variation gene aggregate, per-caller effect rollups, collapsed columns Gene linkage is an aggregate rather than a lookup, so the 25,545 multi-gene loci are correct by construction instead of by a later fix. Effect rollups stay split by caller: snpeff and product_call disagree on 19% of paired calls, 62% of which are product_call's strain-aware downstream_frameshift. Collapsed allele and MAF columns exist only because WDK tables render on record pages, so results pages need pre-aggregated values. --- .../records/variationAttributeQueries.xml | 12 ++++++ .../wdk/model/records/variationRecords.xml | 40 +++++++++++++++++++ 2 files changed, 52 insertions(+) diff --git a/Model/lib/wdk/model/records/variationAttributeQueries.xml b/Model/lib/wdk/model/records/variationAttributeQueries.xml index 593ac64542..d74ffa73f7 100644 --- a/Model/lib/wdk/model/records/variationAttributeQueries.xml +++ b/Model/lib/wdk/model/records/variationAttributeQueries.xml @@ -59,6 +59,14 @@ + + + + + + + + ') AS formatted_organism, va.ncbi_tax_id, va.dataset + , va.gene_ids, va.gene_count + , va.most_severe_impact_snpeff, va.most_severe_impact_product_call + , va.effect_summary_snpeff, va.effect_summary_product_call + , va.collapsed_allele, va.collapsed_minor_allele_frequency FROM jbrestel.VariationAttributes va ]]> diff --git a/Model/lib/wdk/model/records/variationRecords.xml b/Model/lib/wdk/model/records/variationRecords.xml index 39f9100195..40bedab7e2 100644 --- a/Model/lib/wdk/model/records/variationRecords.xml +++ b/Model/lib/wdk/model/records/variationRecords.xml @@ -86,6 +86,46 @@ + + + + + + + + + + + + + + + + + + From e3571e2286a1045e22113d37dd921ddbf204df68 Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Thu, 30 Jul 2026 01:49:18 -0400 Subject: [PATCH 11/98] Fix variation impact sorting and the collapsed-MAF help text Review of the Task 8 commit found two issues. most_severe_impact_snpeff and most_severe_impact_product_call hold text, so WDK sorted them alphabetically (HIGH, LOW, MODERATE, MODIFIER) -- contradicting the severity ranking their own help text advertises. Adds computed rank columns and points sortingColumn at them, entirely within the WDK query so no tuning table change or rebuild is needed. Also completes collapsed_minor_allele_frequency's help text, which named itself derived but omitted the pointer to the canonical SNP Alleles and Indel Alleles sections that the spec requires and collapsed_allele has. --- .../wdk/model/records/variationAttributeQueries.xml | 12 ++++++++++-- Model/lib/wdk/model/records/variationRecords.xml | 9 +++++++-- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/Model/lib/wdk/model/records/variationAttributeQueries.xml b/Model/lib/wdk/model/records/variationAttributeQueries.xml index d74ffa73f7..1f319d66a2 100644 --- a/Model/lib/wdk/model/records/variationAttributeQueries.xml +++ b/Model/lib/wdk/model/records/variationAttributeQueries.xml @@ -61,8 +61,10 @@ - - + + + + @@ -83,6 +85,12 @@ , va.most_severe_impact_snpeff, va.most_severe_impact_product_call , va.effect_summary_snpeff, va.effect_summary_product_call , va.collapsed_allele, va.collapsed_minor_allele_frequency + , CASE va.most_severe_impact_snpeff + WHEN 'HIGH' THEN 4 WHEN 'MODERATE' THEN 3 + WHEN 'LOW' THEN 2 WHEN 'MODIFIER' THEN 1 END AS most_severe_impact_snpeff_rank + , CASE va.most_severe_impact_product_call + WHEN 'HIGH' THEN 4 WHEN 'MODERATE' THEN 3 + WHEN 'LOW' THEN 2 WHEN 'MODIFIER' THEN 1 END AS most_severe_impact_product_call_rank FROM jbrestel.VariationAttributes va ]]> diff --git a/Model/lib/wdk/model/records/variationRecords.xml b/Model/lib/wdk/model/records/variationRecords.xml index 40bedab7e2..2c768763f3 100644 --- a/Model/lib/wdk/model/records/variationRecords.xml +++ b/Model/lib/wdk/model/records/variationRecords.xml @@ -114,6 +114,10 @@ Empty when there is no product call at this locus."/> + + - + + + +
+
+
+
Organism
$$organism$$
+
Location
$$variation_location$$
+
Variant Type
$$variant_type$$
+
Coding
$$is_coding$$
+
Reference Strain
$$reference_strain$$
+
Gene(s)
$$linkedGeneIds$$
+
Most Severe Impact (SnpEff)
$$most_severe_impact_snpeff$$
+
Most Severe Impact (Product Call)
$$most_severe_impact_product_call$$
+
+
+
+
+
SNP Alleles
+
Reference
$$snp_ref_allele$$
+
Major
$$snp_major_allele_and_freq$$
+
Minor
$$snp_minor_allele_and_freq$$
+
Indel Alleles
+
Reference
$$indel_ref_allele$$
+
Major
$$indel_major_allele_and_freq$$
+
Minor
$$indel_minor_allele_and_freq$$
+
Calls
+
Strain Count
$$distinct_strain_count$$
+
Called / No-Call
$$called_strain_count$$ / $$no_call_strain_count$$
+
Call Rate
$$call_rate$$
+
Heterozygous Strains
$$het_strain_count$$
+
+
+
+ + ]]> + + + + + + From dd4efa6bc29fa96b730404372c819c05d524faf6 Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Thu, 30 Jul 2026 06:57:05 -0400 Subject: [PATCH 13/98] Add variation TranscriptProducts record table One row per transcript and observed codon. This is where gene strand lives, since strand is only unambiguous per transcript once a locus can overlap two genes. --- Model/lib/wdk/apiCommonModel.xml | 1 + .../wdk/model/records/variationRecords.xml | 28 +++++++++ .../model/records/variationTableQueries.xml | 60 +++++++++++++++++++ 3 files changed, 89 insertions(+) create mode 100644 Model/lib/wdk/model/records/variationTableQueries.xml diff --git a/Model/lib/wdk/apiCommonModel.xml b/Model/lib/wdk/apiCommonModel.xml index afb78793b8..f3a043c6d0 100644 --- a/Model/lib/wdk/apiCommonModel.xml +++ b/Model/lib/wdk/apiCommonModel.xml @@ -412,6 +412,7 @@ --> + diff --git a/Model/lib/wdk/model/records/variationRecords.xml b/Model/lib/wdk/model/records/variationRecords.xml index e6a1965b74..345fec65ad 100644 --- a/Model/lib/wdk/model/records/variationRecords.xml +++ b/Model/lib/wdk/model/records/variationRecords.xml @@ -248,6 +248,34 @@ summary="organism,variation_location,linkedGeneIds,variant_type,collapsed_allele,collapsed_minor_allele_frequency,most_severe_impact_snpeff,most_severe_impact_product_call,distinct_strain_count" sorting="organism asc,chromosome_order_num asc,location asc"/> + + + + + + + + + + + + + + + + + + + + + + + +
+ diff --git a/Model/lib/wdk/model/records/variationTableQueries.xml b/Model/lib/wdk/model/records/variationTableQueries.xml new file mode 100644 index 0000000000..382fc4a601 --- /dev/null +++ b/Model/lib/wdk/model/records/variationTableQueries.xml @@ -0,0 +1,60 @@ + + + + + + + + Variant_Pf3D7_01_v3_100057 + PlasmoDB + + + + Variant_11L3_v3_26886 + TriTrypDB + + + + Variant_Chr1_A_fumigatus_Af293_1000005 + FungiDB + + + + + + + + + + + + + + + + + + + + + + + + + From 9cc505bd8b2386e6da4a05f888f94450b9b91f86 Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Thu, 30 Jul 2026 07:04:05 -0400 Subject: [PATCH 14/98] Correct the TranscriptProducts strain_count help text The help said "Number of strains observed with this codon". It is not per-codon: it is per amino-acid PRODUCT, repeated across every codon row sharing that product. At Variant_Pf3D7_01_v3_29514 four synonymous V codons (GTA/GTC/GTG/GTT) all read 162 while the single A codon reads 1. Summing per-codon gives 649 strains at a locus whose distinct_strain_count is 160. Verified at scale rather than inferred from one locus: across all 1,192,971 product groups having more than one codon row, strain_count is constant within the group -- zero exceptions. A scientist summing this column would have over-counted badly, so the displayName now says "(per product)" and the help states the repetition explicitly. Co-Authored-By: Claude Opus 5 --- Model/lib/wdk/model/records/variationRecords.xml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Model/lib/wdk/model/records/variationRecords.xml b/Model/lib/wdk/model/records/variationRecords.xml index 345fec65ad..1f828061bf 100644 --- a/Model/lib/wdk/model/records/variationRecords.xml +++ b/Model/lib/wdk/model/records/variationRecords.xml @@ -271,8 +271,11 @@ - + From b19c82d88fd0e814daf4f7d4cc02ab42f4a20360 Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Thu, 30 Jul 2026 07:07:48 -0400 Subject: [PATCH 15/98] Add variation PredictedEffects record table One table with a visible Source column, not two tables, so caller provenance travels with every row and disagreements stay legible. LEFT joins TranscriptAttributes because na_feature_id is null for intergenic calls; an inner join would drop them silently. --- .../wdk/model/records/variationRecords.xml | 25 ++++++++++++++++++ .../model/records/variationTableQueries.xml | 26 +++++++++++++++++++ 2 files changed, 51 insertions(+) diff --git a/Model/lib/wdk/model/records/variationRecords.xml b/Model/lib/wdk/model/records/variationRecords.xml index 1f828061bf..f18c990a73 100644 --- a/Model/lib/wdk/model/records/variationRecords.xml +++ b/Model/lib/wdk/model/records/variationRecords.xml @@ -279,6 +279,31 @@ + + + + + + + + + + + + + +
+ diff --git a/Model/lib/wdk/model/records/variationTableQueries.xml b/Model/lib/wdk/model/records/variationTableQueries.xml index 382fc4a601..25894f40db 100644 --- a/Model/lib/wdk/model/records/variationTableQueries.xml +++ b/Model/lib/wdk/model/records/variationTableQueries.xml @@ -56,5 +56,31 @@ + + + + + + + + + + + + + + + From cf0dd132b858787a5c7a977b0593bc59cd2ee3a3 Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Thu, 30 Jul 2026 07:13:26 -0400 Subject: [PATCH 16/98] Categorize variation record attributes and tables Adds SNP Alleles, Indel Alleles, and Strain Statistics category nodes so the two allele classes read as distinct sections on the record page. Requires wb ontology; wb model alone does not regenerate the OWL and would leave every attribute uncategorized with no error. --- Model/lib/wdk/ontology/individuals.txt | 58 ++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/Model/lib/wdk/ontology/individuals.txt b/Model/lib/wdk/ontology/individuals.txt index 2e80d328a5..7f46a37097 100644 --- a/Model/lib/wdk/ontology/individuals.txt +++ b/Model/lib/wdk/ontology/individuals.txt @@ -2,6 +2,9 @@ eupath/eupath.owl#recordClassName eupath/eupath.owl#targetType eupath/eupath.owl#name EUPATH_0000052 eupath/eupath.owl#shortDisplayName eupath/eupath.owl#description eupath/eupath.owl#geneOrTranscript EUPATH_0000274 eupath/eupath.owl#scope eupath/eupath.owl#scope eupath/eupath.owl#scope GenomicSequencePropertiesCategory GenomicSequencePropertiesCategory Genomic Sequence Properties 5 GenomicSequenceLocationCategory GenomicSequenceLocationCategory Genomic Location 6 +VariationSnpAlleleCategory http://edamontology.org/topic_2885 category VariationSnpAlleleCategory SNP Alleles 1 +VariationIndelAlleleCategory http://edamontology.org/topic_2885 category VariationIndelAlleleCategory Indel Alleles 2 +VariationStrainStatsCategory http://edamontology.org/topic_2885 category VariationStrainStatsCategory Strain Statistics 3 TextCategory TextCategory Text 1 AlignmentsCategory http://edamontology.org/topic_0080 Sequence Analysis AlignmentsCategory BLAT and Blast Alignments CodingPotentialCategory http://edamontology.org/topic_0080 Sequence Analysis CodingPotentialCategory Coding Potential @@ -1098,3 +1101,58 @@ JbrowseRecordClasses.JbrowseGeneRecordClass.location_text GenomicSequenceLocatio JbrowseRecordClasses.JbrowseGeneRecordClass.GOTerms http://edamontology.org/topic_1775 Function analysis JbrowseRecordClasses.JbrowseGeneRecordClass table GOTerms transcript record JbrowseRecordClasses.Jbrowse.GeneRecordClass.GeneTranscripts http://edamontology.org/topic_0114 Gene Structure JbrowseRecordClasses.JbrowseGeneRecordClass table GeneTranscripts gene record +VariationRecordClasses.VariationRecordClass.variation_location GenomicSequenceLocationCategory GenomicSequenceLocationCategory VariationRecordClasses.VariationRecordClass attribute variation_location results record download +VariationRecordClasses.VariationRecordClass.sequence_source_id GenomicSequenceLocationCategory GenomicSequenceLocationCategory VariationRecordClasses.VariationRecordClass attribute sequence_source_id results record download +VariationRecordClasses.VariationRecordClass.location GenomicSequenceLocationCategory GenomicSequenceLocationCategory VariationRecordClasses.VariationRecordClass attribute location results record download +VariationRecordClasses.VariationRecordClass.location_text GenomicSequenceLocationCategory GenomicSequenceLocationCategory VariationRecordClasses.VariationRecordClass attribute location_text record-internal +VariationRecordClasses.VariationRecordClass.chromosome_order_num GenomicSequenceLocationCategory GenomicSequenceLocationCategory VariationRecordClasses.VariationRecordClass attribute chromosome_order_num record-internal +VariationRecordClasses.VariationRecordClass.organism GenomicSequenceLocationCategory GenomicSequenceLocationCategory VariationRecordClasses.VariationRecordClass attribute organism results record download +VariationRecordClasses.VariationRecordClass.organism_text GenomicSequenceLocationCategory GenomicSequenceLocationCategory VariationRecordClasses.VariationRecordClass attribute organism_text record-internal +VariationRecordClasses.VariationRecordClass.formatted_organism GenomicSequenceLocationCategory GenomicSequenceLocationCategory VariationRecordClasses.VariationRecordClass attribute formatted_organism record-internal +VariationRecordClasses.VariationRecordClass.ncbi_tax_id GenomicSequenceLocationCategory GenomicSequenceLocationCategory VariationRecordClasses.VariationRecordClass attribute ncbi_tax_id record download +VariationRecordClasses.VariationRecordClass.dataset GenomicSequenceLocationCategory GenomicSequenceLocationCategory VariationRecordClasses.VariationRecordClass attribute dataset results record download +VariationRecordClasses.VariationRecordClass.variant_type http://edamontology.org/topic_2885 DNA Polymorphism VariationRecordClasses.VariationRecordClass attribute variant_type results record download +VariationRecordClasses.VariationRecordClass.is_coding http://edamontology.org/topic_2885 DNA Polymorphism VariationRecordClasses.VariationRecordClass attribute is_coding results record download +VariationRecordClasses.VariationRecordClass.reference_strain http://edamontology.org/topic_2885 DNA Polymorphism VariationRecordClasses.VariationRecordClass attribute reference_strain results record download +VariationRecordClasses.VariationRecordClass.record_overview http://edamontology.org/topic_2885 DNA Polymorphism VariationRecordClasses.VariationRecordClass attribute record_overview record download +VariationRecordClasses.VariationRecordClass.snp_ref_allele VariationSnpAlleleCategory SNP Alleles VariationRecordClasses.VariationRecordClass attribute snp_ref_allele results record download +VariationRecordClasses.VariationRecordClass.snp_major_allele VariationSnpAlleleCategory SNP Alleles VariationRecordClasses.VariationRecordClass attribute snp_major_allele results record download +VariationRecordClasses.VariationRecordClass.snp_major_allele_frequency VariationSnpAlleleCategory SNP Alleles VariationRecordClasses.VariationRecordClass attribute snp_major_allele_frequency results record download +VariationRecordClasses.VariationRecordClass.snp_major_allele_strain_count VariationSnpAlleleCategory SNP Alleles VariationRecordClasses.VariationRecordClass attribute snp_major_allele_strain_count results record download +VariationRecordClasses.VariationRecordClass.snp_minor_allele VariationSnpAlleleCategory SNP Alleles VariationRecordClasses.VariationRecordClass attribute snp_minor_allele results record download +VariationRecordClasses.VariationRecordClass.snp_minor_allele_frequency VariationSnpAlleleCategory SNP Alleles VariationRecordClasses.VariationRecordClass attribute snp_minor_allele_frequency results record download +VariationRecordClasses.VariationRecordClass.snp_minor_allele_strain_count VariationSnpAlleleCategory SNP Alleles VariationRecordClasses.VariationRecordClass attribute snp_minor_allele_strain_count results record download +VariationRecordClasses.VariationRecordClass.snp_major_genomic_hgvs VariationSnpAlleleCategory SNP Alleles VariationRecordClasses.VariationRecordClass attribute snp_major_genomic_hgvs results record download +VariationRecordClasses.VariationRecordClass.snp_minor_genomic_hgvs VariationSnpAlleleCategory SNP Alleles VariationRecordClasses.VariationRecordClass attribute snp_minor_genomic_hgvs results record download +VariationRecordClasses.VariationRecordClass.snp_major_allele_and_freq VariationSnpAlleleCategory SNP Alleles VariationRecordClasses.VariationRecordClass attribute snp_major_allele_and_freq record download +VariationRecordClasses.VariationRecordClass.snp_minor_allele_and_freq VariationSnpAlleleCategory SNP Alleles VariationRecordClasses.VariationRecordClass attribute snp_minor_allele_and_freq record download +VariationRecordClasses.VariationRecordClass.indel_ref_allele VariationIndelAlleleCategory Indel Alleles VariationRecordClasses.VariationRecordClass attribute indel_ref_allele results record download +VariationRecordClasses.VariationRecordClass.indel_major_allele VariationIndelAlleleCategory Indel Alleles VariationRecordClasses.VariationRecordClass attribute indel_major_allele results record download +VariationRecordClasses.VariationRecordClass.indel_major_allele_frequency VariationIndelAlleleCategory Indel Alleles VariationRecordClasses.VariationRecordClass attribute indel_major_allele_frequency results record download +VariationRecordClasses.VariationRecordClass.indel_major_allele_strain_count VariationIndelAlleleCategory Indel Alleles VariationRecordClasses.VariationRecordClass attribute indel_major_allele_strain_count results record download +VariationRecordClasses.VariationRecordClass.indel_minor_allele VariationIndelAlleleCategory Indel Alleles VariationRecordClasses.VariationRecordClass attribute indel_minor_allele results record download +VariationRecordClasses.VariationRecordClass.indel_minor_allele_frequency VariationIndelAlleleCategory Indel Alleles VariationRecordClasses.VariationRecordClass attribute indel_minor_allele_frequency results record download +VariationRecordClasses.VariationRecordClass.indel_minor_allele_strain_count VariationIndelAlleleCategory Indel Alleles VariationRecordClasses.VariationRecordClass attribute indel_minor_allele_strain_count results record download +VariationRecordClasses.VariationRecordClass.indel_major_genomic_hgvs VariationIndelAlleleCategory Indel Alleles VariationRecordClasses.VariationRecordClass attribute indel_major_genomic_hgvs results record download +VariationRecordClasses.VariationRecordClass.indel_minor_genomic_hgvs VariationIndelAlleleCategory Indel Alleles VariationRecordClasses.VariationRecordClass attribute indel_minor_genomic_hgvs results record download +VariationRecordClasses.VariationRecordClass.indel_major_allele_and_freq VariationIndelAlleleCategory Indel Alleles VariationRecordClasses.VariationRecordClass attribute indel_major_allele_and_freq record download +VariationRecordClasses.VariationRecordClass.indel_minor_allele_and_freq VariationIndelAlleleCategory Indel Alleles VariationRecordClasses.VariationRecordClass attribute indel_minor_allele_and_freq record download +VariationRecordClasses.VariationRecordClass.indel_frame_effect VariationIndelAlleleCategory Indel Alleles VariationRecordClasses.VariationRecordClass attribute indel_frame_effect results record download +VariationRecordClasses.VariationRecordClass.distinct_strain_count VariationStrainStatsCategory Strain Statistics VariationRecordClasses.VariationRecordClass attribute distinct_strain_count results record download +VariationRecordClasses.VariationRecordClass.called_strain_count VariationStrainStatsCategory Strain Statistics VariationRecordClasses.VariationRecordClass attribute called_strain_count results record download +VariationRecordClasses.VariationRecordClass.no_call_strain_count VariationStrainStatsCategory Strain Statistics VariationRecordClasses.VariationRecordClass attribute no_call_strain_count results record download +VariationRecordClasses.VariationRecordClass.call_rate VariationStrainStatsCategory Strain Statistics VariationRecordClasses.VariationRecordClass attribute call_rate results record download +VariationRecordClasses.VariationRecordClass.total_ploidy_count VariationStrainStatsCategory Strain Statistics VariationRecordClasses.VariationRecordClass attribute total_ploidy_count results record download +VariationRecordClasses.VariationRecordClass.het_strain_count VariationStrainStatsCategory Strain Statistics VariationRecordClasses.VariationRecordClass attribute het_strain_count results record download +VariationRecordClasses.VariationRecordClass.ref_allele_frequency VariationStrainStatsCategory Strain Statistics VariationRecordClasses.VariationRecordClass attribute ref_allele_frequency results record download +VariationRecordClasses.VariationRecordClass.gene_ids http://edamontology.org/topic_0199 Genetic Variation VariationRecordClasses.VariationRecordClass attribute gene_ids results record download +VariationRecordClasses.VariationRecordClass.linkedGeneIds http://edamontology.org/topic_0199 Genetic Variation VariationRecordClasses.VariationRecordClass attribute linkedGeneIds results record download +VariationRecordClasses.VariationRecordClass.most_severe_impact_snpeff http://edamontology.org/topic_0199 Genetic Variation VariationRecordClasses.VariationRecordClass attribute most_severe_impact_snpeff results record download +VariationRecordClasses.VariationRecordClass.most_severe_impact_product_call http://edamontology.org/topic_0199 Genetic Variation VariationRecordClasses.VariationRecordClass attribute most_severe_impact_product_call results record download +VariationRecordClasses.VariationRecordClass.effect_summary_snpeff http://edamontology.org/topic_0199 Genetic Variation VariationRecordClasses.VariationRecordClass attribute effect_summary_snpeff results record download +VariationRecordClasses.VariationRecordClass.effect_summary_product_call http://edamontology.org/topic_0199 Genetic Variation VariationRecordClasses.VariationRecordClass attribute effect_summary_product_call results record download +VariationRecordClasses.VariationRecordClass.collapsed_allele http://edamontology.org/topic_0199 Genetic Variation VariationRecordClasses.VariationRecordClass attribute collapsed_allele results record download +VariationRecordClasses.VariationRecordClass.collapsed_minor_allele_frequency http://edamontology.org/topic_0199 Genetic Variation VariationRecordClasses.VariationRecordClass attribute collapsed_minor_allele_frequency results record download +VariationRecordClasses.VariationRecordClass.gene_count http://edamontology.org/topic_0199 Genetic Variation VariationRecordClasses.VariationRecordClass attribute gene_count record-internal +VariationRecordClasses.VariationRecordClass.TranscriptProducts http://edamontology.org/topic_0199 Genetic Variation VariationRecordClasses.VariationRecordClass table TranscriptProducts record download +VariationRecordClasses.VariationRecordClass.PredictedEffects http://edamontology.org/topic_0199 Genetic Variation VariationRecordClasses.VariationRecordClass table PredictedEffects record download From d636b443bd4159493fd29558f6f07530e9616e46 Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Thu, 30 Jul 2026 07:17:45 -0400 Subject: [PATCH 17/98] Label the overview allele rows by class instead of header rows The overview used '
SNP Alleles
' as a section header, but the client DROPS any dt/dd pair whose dd is empty. Both headers therefore vanished, and a MIXED locus rendered two identical Reference/Major/Minor triplets with nothing indicating which was the SNP view and which the indel view -- precisely the ambiguity the two-section design exists to prevent. Verified on Variant_Pf3D7_01_v3_12, which reads: SNP Reference A | SNP Major A (0.9316) | SNP Minor C (0.0598) Indel Reference A | Indel Major A (0.9316) | Indel Minor AC (0.0085) The same empty-dd rule is why Gene(s) and Most Severe Impact (Product Call) are correctly absent on that locus: both are genuinely null there. Co-Authored-By: Claude Opus 5 --- .../wdk/model/records/variationRecords.xml | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/Model/lib/wdk/model/records/variationRecords.xml b/Model/lib/wdk/model/records/variationRecords.xml index f18c990a73..35d6b1adff 100644 --- a/Model/lib/wdk/model/records/variationRecords.xml +++ b/Model/lib/wdk/model/records/variationRecords.xml @@ -219,15 +219,17 @@
-
SNP Alleles
-
Reference
$$snp_ref_allele$$
-
Major
$$snp_major_allele_and_freq$$
-
Minor
$$snp_minor_allele_and_freq$$
-
Indel Alleles
-
Reference
$$indel_ref_allele$$
-
Major
$$indel_major_allele_and_freq$$
-
Minor
$$indel_minor_allele_and_freq$$
-
Calls
+ +
SNP Reference
$$snp_ref_allele$$
+
SNP Major
$$snp_major_allele_and_freq$$
+
SNP Minor
$$snp_minor_allele_and_freq$$
+
Indel Reference
$$indel_ref_allele$$
+
Indel Major
$$indel_major_allele_and_freq$$
+
Indel Minor
$$indel_minor_allele_and_freq$$
Strain Count
$$distinct_strain_count$$
Called / No-Call
$$called_strain_count$$ / $$no_call_strain_count$$
Call Rate
$$call_rate$$
From 51cb41bb0d3e90cb90b4b2acd4ccf91c2a04090b Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Thu, 30 Jul 2026 07:21:55 -0400 Subject: [PATCH 18/98] Assemble the "allele (frequency)" strings in SQL, not as textAttributes A textAttribute template of "$$allele$$ ($$frequency$$)" renders a bare " ()" when both operands are null, so every pure-SNV record page showed "Indel Major ()" and "Indel Minor ()" -- about 4M of the 4.39M records, reading as a rendering bug rather than as "no indel at this locus". Moving the concatenation into SQL with a CASE yields NULL instead, and the client already drops any dt/dd pair whose dd is empty (the same rule that correctly hides Gene(s) on an intergenic locus). Verified on all three variant types via the service: SNV -> snp_* populated, indel_* null INDEL -> indel_* populated, snp_* null MIXED -> both populated and on the rendered SNV page: zero occurrences of "()" in the overview. Co-Authored-By: Claude Opus 5 --- .../records/variationAttributeQueries.xml | 21 ++++++++++++++++++- .../wdk/model/records/variationRecords.xml | 20 +++++++----------- 2 files changed, 28 insertions(+), 13 deletions(-) diff --git a/Model/lib/wdk/model/records/variationAttributeQueries.xml b/Model/lib/wdk/model/records/variationAttributeQueries.xml index 1f319d66a2..607e570a97 100644 --- a/Model/lib/wdk/model/records/variationAttributeQueries.xml +++ b/Model/lib/wdk/model/records/variationAttributeQueries.xml @@ -123,6 +123,10 @@ + + + + @@ -151,7 +155,22 @@ vf.distinct_strain_count, vf.called_strain_count, vf.no_call_strain_count, vf.call_rate, vf.total_ploidy_count, vf.het_strain_count, - vf.ref_allele_frequency + vf.ref_allele_frequency, + -- "allele (frequency)" assembled here rather than in a textAttribute, whose + -- template would render a bare " ()" for the absent allele class. NULL lets + -- the client drop the row entirely. + CASE WHEN vf.snp_major_allele IS NOT NULL + THEN concat(vf.snp_major_allele, ' (', vf.snp_major_allele_frequency, ')') + END AS snp_major_allele_and_freq, + CASE WHEN vf.snp_minor_allele IS NOT NULL + THEN concat(vf.snp_minor_allele, ' (', vf.snp_minor_allele_frequency, ')') + END AS snp_minor_allele_and_freq, + CASE WHEN vf.indel_major_allele IS NOT NULL + THEN concat(vf.indel_major_allele, ' (', vf.indel_major_allele_frequency, ')') + END AS indel_major_allele_and_freq, + CASE WHEN vf.indel_minor_allele IS NOT NULL + THEN concat(vf.indel_minor_allele, ' (', vf.indel_minor_allele_frequency, ')') + END AS indel_minor_allele_and_freq FROM apidb.VariationFeature vf, jbrestel.VariationAttributes va WHERE va.source_id = vf.source_id diff --git a/Model/lib/wdk/model/records/variationRecords.xml b/Model/lib/wdk/model/records/variationRecords.xml index 35d6b1adff..40af014865 100644 --- a/Model/lib/wdk/model/records/variationRecords.xml +++ b/Model/lib/wdk/model/records/variationRecords.xml @@ -152,12 +152,12 @@ help="Genomic HGVS for the major allele. Empty when the major allele is identical to the reference allele, which is the usual case."/> - - - - - - + + + @@ -172,12 +172,8 @@ is identical to the reference allele, which is the usual case."/> - - - - - - + + - - + + @@ -172,8 +173,8 @@ is identical to the reference allele, which is the usual case."/> - - + + - + Indel Reference
$$indel_ref_allele$$
Indel Major
$$indel_major_allele_and_freq$$
Indel Minor
$$indel_minor_allele_and_freq$$
-
Strain Count
$$distinct_strain_count$$
+
Called Strain Count (including reference)
$$distinct_strain_count$$
Called / No-Call
$$called_strain_count$$ / $$no_call_strain_count$$
Call Rate
$$call_rate$$
Heterozygous Strains
$$het_strain_count$$
From 2723cdac2b8cf8c882845986b3ddacf49dc0cc36 Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Thu, 30 Jul 2026 12:44:18 -0400 Subject: [PATCH 21/98] Point the variation record at the real tuning table Replaces the temporary jbrestel.VariationAttributes stub with ApidbTuning.VariationAttributes now that the tuning job has built it, and removes the TEMPORARY STUB comment from both query files. 6 SQL references flipped: 4 in variationAttributeQueries.xml, 2 in variationTableQueries.xml. The real table was compared against the stub before flipping rather than assumed equivalent: identical column sets and ordering, and EXCEPT in both directions returned 0 rows over all 4,390,908 rows. Aggregates populated as expected (gene_ids on 2,879,337 loci -- zero would have meant the TranscriptAttributes dependency failed to resolve). Record pages verified unchanged across all five test loci: SNV, INDEL, MIXED (both allele classes), multi-gene, and a caller-disagreement locus. Assembled SQL confirmed to read ApidbTuning via wdkQuery -showQuery. grep -rn jbrestel over Model/lib/wdk and Model/lib/xml returns nothing. Co-Authored-By: Claude Opus 5 --- .../wdk/model/records/variationAttributeQueries.xml | 12 ++++-------- .../lib/wdk/model/records/variationTableQueries.xml | 8 ++------ 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/Model/lib/wdk/model/records/variationAttributeQueries.xml b/Model/lib/wdk/model/records/variationAttributeQueries.xml index 607e570a97..d73e3fba92 100644 --- a/Model/lib/wdk/model/records/variationAttributeQueries.xml +++ b/Model/lib/wdk/model/records/variationAttributeQueries.xml @@ -1,9 +1,5 @@ - - @@ -23,7 +19,7 @@ - SELECT count(*) FROM jbrestel.VariationAttributes + SELECT count(*) FROM ApidbTuning.VariationAttributes - @@ -47,7 +43,7 @@ CASE p.matches_ref_product WHEN 1 THEN 'yes' ELSE 'no' END AS matches_ref_product, p.strain_count, p.hgvs_p FROM apidb.VariationTranscriptProduct p, - jbrestel.VariationAttributes va, + ApidbTuning.VariationAttributes va, ApidbTuning.TranscriptAttributes t WHERE va.sequence_source_id = p.sequence_source_id AND va.location = p.location @@ -73,7 +69,7 @@ t.gene_source_id, t.transcript_source_id, e.impact, e.effect, e.hgvs_c, e.source FROM apidb.VariationEffect e - JOIN jbrestel.VariationAttributes va + JOIN ApidbTuning.VariationAttributes va ON va.sequence_source_id = e.sequence_source_id AND va.location = e.location LEFT JOIN ApidbTuning.TranscriptAttributes t From d3616c69e707a5e53c23a725c229f63773f1b19e Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Wed, 5 Aug 2026 09:19:19 -0400 Subject: [PATCH 22/98] Add the variation record design spec and implementation plan These were authored in the agentic-veupath-dev control-plane checkout because that is where the build commands run from, but they describe work that lands entirely here. Moved to sit beside the code they specify, matching the convention already used for the EDA data tables and the fungidb strain-segment record. Content is carried over verbatim; the plan's progress block is stale relative to the commits on this branch. Co-Authored-By: Claude Opus 5 --- .../plans/2026-07-30-variation-record.md | 2225 +++++++++++++++++ .../2026-07-30-variation-record-design.md | 518 ++++ 2 files changed, 2743 insertions(+) create mode 100644 docs/superpowers/plans/2026-07-30-variation-record.md create mode 100644 docs/superpowers/specs/2026-07-30-variation-record-design.md diff --git a/docs/superpowers/plans/2026-07-30-variation-record.md b/docs/superpowers/plans/2026-07-30-variation-record.md new file mode 100644 index 0000000000..9e34710668 --- /dev/null +++ b/docs/superpowers/plans/2026-07-30-variation-record.md @@ -0,0 +1,2225 @@ +# Variation Record Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Build a new WDK `variation` record — one record per variant locus — with the attributes, tuning table, and record-page tables specified in `docs/superpowers/specs/2026-07-30-variation-record-design.md`. + +**Architecture:** A thin `apidbtuning.VariationAttributes` tuning table supplies derived, aggregated, and join-requiring columns; `apidb.VariationFeature` is queried directly for intrinsic per-locus facts (it is already one row per locus with a unique `source_id`). Two record-page tables read `apidb.VariationTranscriptProduct` and `apidb.VariationEffect`. Work proceeds as a vertical slice: a minimal buildable record first (Task 4), then attributes and tables added incrementally, each verified by a real build against a live instance. + +**Tech Stack:** WDK model XML, EuPathDB tuning manager XML, PostgreSQL, `wb` build wrapper via `bin/veup-build.sh`, `wdkQuery` for SQL introspection, WDK REST service for verification. + +--- + +## Context you need before starting + +> ### Progress +> +> **Tasks 0-4 are done. Task 5 is next.** +> +> Task 4 committed as `9bcf147` in `ApiCommonModel`: the record builds green, 13 +> attributes are registered, and +> `/plasmo.jbrestel/app/record/variation/Variant_Pf3D7_01_v3_29514` renders as +> "Variation: Variant_Pf3D7_01_v3_29514" with every service call 200 and all error logs +> silent. Building it surfaced three defects in this plan, now fixed: the app/service base +> URLs both include `/plasmo.jbrestel`, an empty `querySet` is invalid so +> `variationTableQueries.xml` moved to Task 10, and the snp imports sit inside a comment +> block that would have swallowed the new imports. +> +> **Tasks 0-3 detail.** +> +> - Tuning definition committed to `ApiCommonModel` on `dnaseq-merge-experiments` as +> `9ff6094` (`apiTuningManager.xml`, +103 lines, XML validated, tuning-table count +> 54 → 55). +> - `jbrestel.VariationAttributes` built and verified: **4,390,908 rows** (exactly +> matching `apidb.VariationFeature`) in 66s, three indexes, `GRANT SELECT TO gus_r`. +> `gene_ids` populated for 2,879,337 loci, `most_severe_impact_snpeff` for 4,390,895, +> `most_severe_impact_product_call` for 1,690,908, `collapsed_allele` for all +> 4,390,908, 3 projects, **25,545 multi-gene loci** — matching the spec's figure +> exactly. All three spot-check loci correct, including `A>C; A>AC` for the MIXED locus. +> +> ### ⚠ The model reads a developer-schema stub +> +> `tuningManager` is **not installed** on this dev instance, so the table lives in +> `jbrestel.VariationAttributes` rather than `apidbtuning`. Tasks 4-13 therefore write +> `jbrestel.VariationAttributes` into the query XML in **six places**. +> +> **This must not reach a merge.** Task 14 flips it back and fails loudly if any stub +> reference survives. Do not skip it. + +**Two repos are involved:** + +| repo | path | role | +|---|---|---| +| `ApiCommonModel` | `~/workspaces/plasmodb/ApiCommonModel` | all edits land here; branch `dnaseq-merge-experiments` | +| `agentic-veupath-dev` | `~/workspaces/agentic-veupath-dev` | control plane — run builds from here | + +**Concrete instance values** (resolved from `profiles/plasmodb.yml` + `profiles/identity.yml`): + +| value | | +|---|---| +| ssh host | `cedar` | +| docroot | `/var/www/jbrestel.plasmodb.org/project_home` | +| setenv | `/var/www/jbrestel.plasmodb.org/etc/setenv` | +| app URL | `https://jbrestel.plasmodb.org/plasmo.jbrestel/app` | +| service base | `/plasmo.jbrestel/service` (alias `/a/service`) — **not** `/service` | +| GUS project | `PlasmoDB` | +| local appDb (psql) | `unidb_shu_a` on `localhost:5432` | + +**Commands you will use repeatedly:** + +```bash +# Build the WDK model (questions, queries, records). Run from the harness repo. +cd ~/workspaces/agentic-veupath-dev && bin/veup-build.sh plasmodb wb model + +# Rebuild the category OWL *and* the model. Required for any individuals.txt change. +cd ~/workspaces/agentic-veupath-dev && bin/veup-build.sh plasmodb wb ontology + +# Render a WDK query's assembled SQL without executing it (always safe). +ssh cedar 'bash -lc "source /var/www/jbrestel.plasmodb.org/etc/setenv && \ + wdkQuery -model PlasmoDB -query -showQuery"' + +# Read remote logs by page-load delta. +cd ~/workspaces/agentic-veupath-dev && bin/veup-logs.sh plasmodb mark `: + +```xml + + + + + + + + + + + +``` + +- [ ] **Step 3: Add the attributes to `variationRecords.xml`** + +Insert after the closing `` of the `VariationTuning` block: + +```xml + + + + + +``` + +- [ ] **Step 4: Build** + +```bash +cd ~/workspaces/agentic-veupath-dev && bin/veup-build.sh plasmodb wb model +``` + +Expected: green build. + +- [ ] **Step 5: Verify on the record page** + +Reload `https://jbrestel.plasmodb.org/plasmo.jbrestel/app/record/variation/Variant_Pf3D7_01_v3_29514` + +Expected: Variant Type `SNV`, Coding `coding`, Reference Strain `3D7`. + +- [ ] **Step 6: Commit** + +```bash +cd ~/workspaces/plasmodb/ApiCommonModel +git add Model/lib/wdk/model/records/variationAttributeQueries.xml \ + Model/lib/wdk/model/records/variationRecords.xml +git commit -m "Add variation classification attributes from VariationFeature + +First attribute query reading apidb.VariationFeature directly, per the +sourcing rule: intrinsic per-locus facts come from the base table, only +derived/joined columns come from the tuning table." +``` + +--- + +## Task 6: SNP and Indel allele sections + +The core of the design: two named sections, never collapsed. + +**Files:** +- Modify: `Model/lib/wdk/model/records/variationAttributeQueries.xml` +- Modify: `Model/lib/wdk/model/records/variationRecords.xml` + +- [ ] **Step 1: Assert the MIXED locus carries both classes** + +```bash +psql -h localhost -p 5432 -d unidb_shu_a -x -c " +select snp_ref_allele, snp_major_allele, snp_minor_allele, snp_minor_allele_frequency, + indel_ref_allele, indel_major_allele, indel_minor_allele, indel_minor_allele_frequency, + snp_minor_genomic_hgvs, indel_minor_genomic_hgvs, indel_frame_effect +from apidb.variationfeature where source_id = 'Variant_Pf3D7_01_v3_12'" +``` + +Expected: SNP side `A`/`A`/`C`/`0.0598`, indel side `A`/`A`/`AC`/`0.0085`, +`snp_minor_genomic_hgvs` = `Pf3D7_01_v3:g.12A>C`, `indel_minor_genomic_hgvs` = +`Pf3D7_01_v3:g.12_13insC`. Both sides populated on one row is exactly the case that +motivates two sections. + +- [ ] **Step 2: Extend `VariationFeatureBase` with all 19 allele columns** + +19, not 22: 9 `snp_*` + 10 `indel_*`. `indel_frame_effect` is indel-only, so the two +sections are deliberately asymmetric. + +Add these `` declarations to the existing `VariationFeatureBase` query, after +``: + +```xml + + + + + + + + + + + + + + + + + + + +``` + +And add the matching columns to the `SELECT` list, immediately after +`vf.reference_strain,`: + +```sql + vf.snp_ref_allele, vf.snp_major_allele, + vf.snp_major_allele_frequency, vf.snp_major_allele_strain_count, + vf.snp_minor_allele, + vf.snp_minor_allele_frequency, vf.snp_minor_allele_strain_count, + vf.snp_major_genomic_hgvs, vf.snp_minor_genomic_hgvs, + vf.indel_ref_allele, vf.indel_major_allele, + vf.indel_major_allele_frequency, vf.indel_major_allele_strain_count, + vf.indel_minor_allele, + vf.indel_minor_allele_frequency, vf.indel_minor_allele_strain_count, + vf.indel_major_genomic_hgvs, vf.indel_minor_genomic_hgvs, + vf.indel_frame_effect, +``` + +- [ ] **Step 3: Add the allele attributes to `variationRecords.xml`** + +Add inside the existing `VariationFeatureBase` `` block, after +`reference_strain`. Note the shared `help` text on the HGVS attributes — the +major-allele HGVS is null whenever the major allele equals the reference, and without +this it reads as missing data. + +```xml + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +``` + +- [ ] **Step 4: Build** + +```bash +cd ~/workspaces/agentic-veupath-dev && bin/veup-build.sh plasmodb wb model +``` + +Expected: green build. + +- [ ] **Step 5: Verify all three variant types render correctly** + +Load each and confirm: + +| record | expect | +|---|---| +| `Variant_Pf3D7_01_v3_29514` (SNV) | SNP attributes populated; indel attributes empty | +| `Variant_Pf3D7_01_v3_18` (INDEL) | indel populated (`AC` / `A`); SNP empty | +| `Variant_Pf3D7_01_v3_12` (MIXED) | **both** populated — SNP `A`/`C`, indel `A`/`AC` | + +The MIXED case is the acceptance test for this task. + +- [ ] **Step 6: Commit** + +```bash +cd ~/workspaces/plasmodb/ApiCommonModel +git add Model/lib/wdk/model/records/variationAttributeQueries.xml \ + Model/lib/wdk/model/records/variationRecords.xml +git commit -m "Add SNP and Indel allele attributes as separate sections + +Both allele classes are preserved rather than collapsed: 129,850 MIXED +loci populate snp_* and indel_* simultaneously and describe the same +strains two ways, so a single major/minor pair would silently lose half +the data. + +HGVS attributes carry help text explaining that the major-allele HGVS is +empty whenever the major allele equals the reference." +``` + +--- + +## Task 7: Strain and call statistics + +**Files:** +- Modify: `Model/lib/wdk/model/records/variationAttributeQueries.xml` +- Modify: `Model/lib/wdk/model/records/variationRecords.xml` + +- [ ] **Step 1: Assert the values in psql** + +```bash +psql -h localhost -p 5432 -d unidb_shu_a -x -c " +select distinct_strain_count, called_strain_count, no_call_strain_count, call_rate, + total_ploidy_count, het_strain_count, ref_allele_frequency +from apidb.variationfeature where source_id = 'Variant_Pf3D7_01_v3_29514'" +``` + +Expected: `160`, `159`, `57`, `0.7361`, `160`, `0`, `0.9938`. + +- [ ] **Step 2: Add columns to `VariationFeatureBase`** + +Add the `` declarations: + +```xml + + + + + + + +``` + +And to the `SELECT` list. + +**Mind the comma.** After Task 6 the `SELECT` ends with `vf.indel_frame_effect` and **no** +trailing comma, immediately followed by `FROM apidb.VariationFeature vf,`. So you must add +a comma to the current last line, and your own last line must NOT have one. The tail +should read exactly: + +```sql + vf.indel_frame_effect, + vf.distinct_strain_count, vf.called_strain_count, + vf.no_call_strain_count, vf.call_rate, + vf.total_ploidy_count, vf.het_strain_count, + vf.ref_allele_frequency + FROM apidb.VariationFeature vf, + jbrestel.VariationAttributes va + WHERE va.source_id = vf.source_id +``` + +- [ ] **Step 3: Add the attributes to `variationRecords.xml`** + +Inside the same `VariationFeatureBase` `` block: + +```xml + + + + + + + + +``` + +- [ ] **Step 4: Build** + +```bash +cd ~/workspaces/agentic-veupath-dev && bin/veup-build.sh plasmodb wb model +``` + +Expected: green build. + +- [ ] **Step 5: Verify** + +Load `https://jbrestel.plasmodb.org/plasmo.jbrestel/app/record/variation/Variant_Pf3D7_01_v3_29514` and +confirm Strain Count `160`, Call Rate `0.7361`, Reference Allele Frequency `0.9938`. + +- [ ] **Step 6: Commit** + +```bash +cd ~/workspaces/plasmodb/ApiCommonModel +git add Model/lib/wdk/model/records/variationAttributeQueries.xml \ + Model/lib/wdk/model/records/variationRecords.xml +git commit -m "Add variation strain and call statistic attributes + +Grouped together deliberately: these aggregates are what the future +per-strain VCF-backed table will detail, so it lands beside them." +``` + +--- + +## Task 8: Gene linkage, effect rollups, and collapsed columns + +**Files:** +- Modify: `Model/lib/wdk/model/records/variationAttributeQueries.xml` +- Modify: `Model/lib/wdk/model/records/variationRecords.xml` + +- [ ] **Step 1: Assert a multi-gene locus exists and find one** + +```bash +psql -h localhost -p 5432 -d unidb_shu_a -c " +select source_id, gene_ids, gene_count, most_severe_impact_snpeff, + most_severe_impact_product_call, collapsed_allele +from jbrestel.variationattributes +where gene_count > 1 and project_id = 'PlasmoDB' order by source_id limit 3" +``` + +**Filter to `project_id = 'PlasmoDB'`** — you are verifying against a PlasmoDB instance, +and a TriTrypDB or FungiDB record will not resolve there. Multi-gene loci are distributed +TriTrypDB 16,189 / PlasmoDB 4,698 / FungiDB 4,658, so an unfiltered `order by source_id +limit 3` returns TriTrypDB records that 404 on this site. + +Verified PlasmoDB multi-gene loci, usable directly: + +| `source_id` | `gene_ids` | `gene_count` | `collapsed_allele` | +|---|---|---|---| +| `Variant_Pf3D7_01_v3_125983` | `PF3D7_0102700, PF3D7_0102800` | 2 | `AAT>AATATAT` | +| `Variant_Pf3D7_01_v3_126183` | `PF3D7_0102700, PF3D7_0102800` | 2 | `TA>T` | +| `Variant_Pf3D7_01_v3_126195` | `PF3D7_0102700, PF3D7_0102800` | 2 | `A>C` | + +If the query returns nothing, the gene aggregate is broken; go back to Task 3. + +- [ ] **Step 2: Extend `VariationTuning` with the aggregate columns** + +Add the `` declarations to the existing `VariationTuning` query: + +```xml + + + + + + + + + + +``` + +And to its `SELECT`, after `va.dataset`. This snippet uses **leading** commas, so it +appends cleanly with no trailing-comma hazard — the tail should read: + +```sql + va.ncbi_tax_id, + va.dataset + , va.gene_ids, va.gene_count + , va.most_severe_impact_snpeff, va.most_severe_impact_product_call + , va.effect_summary_snpeff, va.effect_summary_product_call + , va.collapsed_allele, va.collapsed_minor_allele_frequency + , CASE va.most_severe_impact_snpeff + WHEN 'HIGH' THEN 4 WHEN 'MODERATE' THEN 3 + WHEN 'LOW' THEN 2 WHEN 'MODIFIER' THEN 1 END AS most_severe_impact_snpeff_rank + , CASE va.most_severe_impact_product_call + WHEN 'HIGH' THEN 4 WHEN 'MODERATE' THEN 3 + WHEN 'LOW' THEN 2 WHEN 'MODIFIER' THEN 1 END AS most_severe_impact_product_call_rank + FROM jbrestel.VariationAttributes va +``` + +**Why the rank columns.** The impact columns hold the *text* `HIGH`/`MODERATE`/`LOW`/`MODIFIER`, +so without a `sortingColumn` WDK sorts them alphabetically — `HIGH, LOW, MODERATE, MODIFIER` — +directly contradicting the severity ranking the columns' own `help` text advertises. A user +sorting a results page by impact would silently get a wrong order. `sortingColumn` only needs +a column returned by the *same query*, so a `CASE` here fixes it with no tuning-table change +and no rebuild. + +- [ ] **Step 3: Add the attributes to `variationRecords.xml`** + +Inside the existing `VariationTuning` `` block: + +```xml + + + + + + + + + + + + + + + + + + + + + +``` + +**Note on `linkedGeneIds`:** WDK's `linkAttribute` builds exactly one URL, so it cannot +render one link per ID in an aggregated string. A `textAttribute` is used instead, and it +renders IDs as plain text. If per-ID hyperlinks are wanted, that needs a client-side +component and should be raised with the user as a follow-up rather than faked here. + +- [ ] **Step 4: Build** + +```bash +cd ~/workspaces/agentic-veupath-dev && bin/veup-build.sh plasmodb wb model +``` + +Expected: green build. + +- [ ] **Step 5: Verify single-gene and multi-gene loci both render** + +- `Variant_Pf3D7_01_v3_29514` → Gene ID(s) `PF3D7_0100100`, Gene Count `1`, Allele `T>C`, + both Most Severe Impact columns `MODERATE`. +- `Variant_Pf3D7_01_v3_12` → Allele `A>C; A>AC`, Minor Allele Frequency `0.0598`, + SnpEff impact `MODIFIER`, Product Call impact **empty**. +- `Variant_Pf3D7_01_v3_126195` → Gene ID(s) shows **two** comma-separated IDs + (`PF3D7_0102700, PF3D7_0102800`) and Gene Count `2`. This is the acceptance test: it is + the case a single-gene lookup would have silently misreported. + +- [ ] **Step 6: Commit** + +```bash +cd ~/workspaces/plasmodb/ApiCommonModel +git add Model/lib/wdk/model/records/variationAttributeQueries.xml \ + Model/lib/wdk/model/records/variationRecords.xml +git commit -m "Add variation gene aggregate, per-caller effect rollups, collapsed columns + +Gene linkage is an aggregate rather than a lookup, so the 25,545 +multi-gene loci are correct by construction instead of by a later fix. + +Effect rollups stay split by caller: snpeff and product_call disagree on +19% of paired calls, 62% of which are product_call's strain-aware +downstream_frameshift. + +Collapsed allele and MAF columns exist only because WDK tables render on +record pages, so results pages need pre-aggregated values." +``` + +--- + +## Task 9: Record overview and default summary + +**Files:** +- Modify: `Model/lib/wdk/model/records/variationRecords.xml` + +- [ ] **Step 1: Add the overview text attribute** + +Insert immediately before the closing ``. Both allele sections appear, each +rendered unconditionally — WDK text attributes have no conditionals, so an absent class +shows empty values, which is honest and matches how the `snp` record behaved. + +```xml + + + +
+
+
+
Organism
$$organism$$
+
Location
$$variation_location$$
+
Variant Type
$$variant_type$$
+
Coding
$$is_coding$$
+
Reference Strain
$$reference_strain$$
+
Gene(s)
$$linkedGeneIds$$
+
Most Severe Impact (SnpEff)
$$most_severe_impact_snpeff$$
+
Most Severe Impact (Product Call)
$$most_severe_impact_product_call$$
+
+
+
+
+
SNP Alleles
+
Reference
$$snp_ref_allele$$
+
Major
$$snp_major_allele_and_freq$$
+
Minor
$$snp_minor_allele_and_freq$$
+
Indel Alleles
+
Reference
$$indel_ref_allele$$
+
Major
$$indel_major_allele_and_freq$$
+
Minor
$$indel_minor_allele_and_freq$$
+
Calls
+
Strain Count
$$distinct_strain_count$$
+
Called / No-Call
$$called_strain_count$$ / $$no_call_strain_count$$
+
Call Rate
$$call_rate$$
+
Heterozygous Strains
$$het_strain_count$$
+
+
+
+
+ ]]> + + +``` + +- [ ] **Step 2: Replace the placeholder `attributesList`** + +Replace the `` line added in Task 4 with: + +```xml + + + +``` + +- [ ] **Step 3: Build** + +```bash +cd ~/workspaces/agentic-veupath-dev && bin/veup-build.sh plasmodb wb model +``` + +Expected: green build. A failure naming an attribute in `summary` means it is misspelled +or not defined — every name in `attributesList` must exist. + +- [ ] **Step 4: Verify the overview renders on the MIXED locus** + +Load `https://jbrestel.plasmodb.org/plasmo.jbrestel/app/record/variation/Variant_Pf3D7_01_v3_12` + +Expected: a two-panel overview; the right panel shows **both** a SNP Alleles block +(`A`, minor `C (0.0598)`) and an Indel Alleles block (`A`, minor `AC (0.0085)`). + +- [ ] **Step 5: Check the logs are clean** + +```bash +cd ~/workspaces/agentic-veupath-dev && bin/veup-logs.sh plasmodb mark overview +# reload the record page in the browser, let it settle +cd ~/workspaces/agentic-veupath-dev && bin/veup-logs.sh plasmodb since overview --quiet +``` + +Expected: error logs reported as `silent:`. An unresolved `$$attribute$$` shows up here +rather than on the page. + +- [ ] **Step 6: Commit** + +```bash +cd ~/workspaces/plasmodb/ApiCommonModel +git add Model/lib/wdk/model/records/variationRecords.xml +git commit -m "Add variation record overview and default summary columns + +Overview shows both allele sections so a MIXED locus reads honestly. +Summary columns use the collapsed allele and MAF, since results pages +cannot render tables." +``` + +--- + +## Task 10: TranscriptProducts table + +**Files:** +- **Create:** `Model/lib/wdk/model/records/variationTableQueries.xml` (does not exist yet — an + empty `querySet` is invalid, so the file arrives with this first query) +- Modify: `Model/lib/wdk/apiCommonModel.xml` (add its import) +- Modify: `Model/lib/wdk/model/records/variationRecords.xml` + +- [ ] **Step 1: Assert the expected rows in psql** + +```bash +psql -h localhost -p 5432 -d unidb_shu_a -c " +select t.gene_source_id, t.transcript_source_id, t.strand, + p.pos_in_cds, p.pos_in_protein, p.codon, p.pos_in_codon, p.product, + p.matches_ref_codon, p.matches_ref_product, p.strain_count, p.hgvs_p +from apidb.variationtranscriptproduct p +join apidbtuning.transcriptattributes t on t.na_feature_id = p.na_feature_id +where p.sequence_source_id = 'Pf3D7_01_v3' and p.location = 29514 +order by p.codon" +``` + +Expected: **5 rows**, all gene `PF3D7_0100100`, codons `GCG`, `GTA`, `GTC`, `GTG`, `GTT`; +the `GCG` row has product `A` with `matches_ref_product` `0`, the rest product `V` with +`1`. + +- [ ] **Step 2: Create `variationTableQueries.xml` with this query** + +Full file. The stub comment matters — this file also reads the developer-schema table. + +```xml + + + + + + + + Variant_Pf3D7_01_v3_100057 + PlasmoDB + + + + Variant_11L3_v3_26886 + TriTrypDB + + + + Variant_Chr1_A_fumigatus_Af293_1000005 + FungiDB + + + + + + + + + + + + + + + + + + + + + + + + + +``` + +- [ ] **Step 2a: Add its import to `apiCommonModel.xml`** + +Between the two existing variation imports, so query sets precede the record class: + +```xml + + + +``` + +Then re-run the ACTIVE check from Task 4 Step 4, now expecting all three `ACTIVE`. + +- [ ] **Step 3: Add the table to `variationRecords.xml`** + +Insert immediately before the closing ``: + +```xml + + + + + + + + + + + + + + + + + + + + + + + +
+``` + +- [ ] **Step 4: Build** + +```bash +cd ~/workspaces/agentic-veupath-dev && bin/veup-build.sh plasmodb wb model +``` + +Expected: green build. + +- [ ] **Step 5: Verify the assembled SQL, then the page** + +```bash +ssh cedar 'bash -lc "source /var/www/jbrestel.plasmodb.org/etc/setenv && \ + wdkQuery -model PlasmoDB -query VariationTables.TranscriptProducts -showQuery"' +``` + +Expected: the three-way join, unchanged. + +Then load +`https://jbrestel.plasmodb.org/plasmo.jbrestel/app/record/variation/Variant_Pf3D7_01_v3_29514` and +confirm the table shows the **5 codon rows** from Step 1, with the `GCG`/`A` row marked +`no` for Matches Reference Product. + +- [ ] **Step 6: Commit** + +```bash +cd ~/workspaces/plasmodb/ApiCommonModel +git add Model/lib/wdk/model/records/variationTableQueries.xml \ + Model/lib/wdk/model/records/variationRecords.xml \ + Model/lib/wdk/apiCommonModel.xml +git commit -m "Add variation TranscriptProducts record table + +One row per transcript and observed codon. This is where gene strand +lives, since strand is only unambiguous per transcript once a locus can +overlap two genes." +``` + +--- + +## Task 11: PredictedEffects table + +**Files:** +- Modify: `Model/lib/wdk/model/records/variationTableQueries.xml` +- Modify: `Model/lib/wdk/model/records/variationRecords.xml` + +- [ ] **Step 1: Assert both callers appear, including an intergenic locus** + +```bash +psql -h localhost -p 5432 -d unidb_shu_a -c " +select allele, na_feature_id, impact, effect, hgvs_c, source +from apidb.variationeffect +where sequence_source_id='Pf3D7_01_v3' and location in (12, 29514) +order by location, source, allele" +``` + +Expected: for 12, two `snpeff` rows with **null** `na_feature_id` and effect +`intergenic_region`; for 29514, one `snpeff` and one `product_call` row, both +`MODERATE`/`missense_variant`/`c.5T>C`. The null `na_feature_id` is why the join must be +a LEFT join. + +- [ ] **Step 2: Add the query to `variationTableQueries.xml`** + +```xml + + + + + + + + + + + + + + +``` + +The LEFT join is required: `na_feature_id` is null for intergenic calls, and an inner +join would silently drop every intergenic effect row. + +- [ ] **Step 3: Add the table to `variationRecords.xml`** + +Insert before the closing ``: + +```xml + + + + + + + + + + + + + +
+``` + +- [ ] **Step 4: Build** + +```bash +cd ~/workspaces/agentic-veupath-dev && bin/veup-build.sh plasmodb wb model +``` + +Expected: green build. + +- [ ] **Step 5: Verify both callers and the intergenic case render** + +- `Variant_Pf3D7_01_v3_29514` → 2 rows, Source `snpeff` and `product_call`, both + `MODERATE` / `missense_variant`. +- `Variant_Pf3D7_01_v3_12` → 2 `snpeff` rows, effect `intergenic_region`, Gene ID + **empty**. If this table is empty, the LEFT join was written as an inner join. + +- [ ] **Step 6: Find and verify a disagreement locus — the real acceptance test** + +```bash +psql -h localhost -p 5432 -d unidb_shu_a -c " +with p as (select * from apidb.variationeffect where source='product_call'), + s as (select * from apidb.variationeffect where source='snpeff') +select va.source_id, p.effect as product_call_effect, s.effect as snpeff_effect +from p join s using (sequence_source_id, location, allele, na_feature_id) +join jbrestel.variationattributes va + on va.sequence_source_id = p.sequence_source_id and va.location = p.location +where p.effect <> s.effect and p.effect = 'downstream_frameshift' +limit 3" +``` + +Load one of those records and confirm the table shows **both** rows with different +`Effect` values side by side. This is the behaviour the whole two-caller design exists to +produce. + +- [ ] **Step 7: Commit** + +```bash +cd ~/workspaces/plasmodb/ApiCommonModel +git add Model/lib/wdk/model/records/variationTableQueries.xml \ + Model/lib/wdk/model/records/variationRecords.xml +git commit -m "Add variation PredictedEffects record table + +One table with a visible Source column, not two tables, so caller +provenance travels with every row and disagreements stay legible. + +LEFT joins TranscriptAttributes because na_feature_id is null for +intergenic calls; an inner join would drop them silently." +``` + +--- + +## Task 12: Category ontology placement + +**Files:** +- Modify: `Model/lib/wdk/ontology/individuals.txt` + +**Critical:** this is a categorization change, so it needs `wb ontology`, **not** +`wb model`. `wb model` does not regenerate the category OWL, and skipping `wb ontology` +leaves every new attribute uncategorized with **no error anywhere**. + +- [ ] **Step 1: Understand the file format** + +Tab-separated, no quoting. Columns, in order: + +``` +individualIri parentIri parentLabel recordClassName targetType name +displayName shortDisplayName description geneOrTranscript displayOrder +scope scope scope +``` + +Trailing empty columns are still tab-delimited. Study lines 684-692 (the existing +`SnpRecordClasses.SnpRecordClass` attribute entries) before editing — match their exact +tab count. + +- [ ] **Step 2: Add three new category nodes** + +Add near the other category-node definitions at the top of the file (compare +`GenomicSequencePropertiesCategory` on line 3). These are the parents for the allele and +statistics groups: + +``` +VariationSnpAlleleCategory http://edamontology.org/topic_2885 category VariationSnpAlleleCategory SNP Alleles 1 +VariationIndelAlleleCategory http://edamontology.org/topic_2885 category VariationIndelAlleleCategory Indel Alleles 2 +VariationStrainStatsCategory http://edamontology.org/topic_2885 category VariationStrainStatsCategory Strain Statistics 3 +``` + +- [ ] **Step 3: Add attribute and table entries** + +Add one line per attribute and table. The parent assignments follow spec §8: + +| group | parent | +|---|---| +| identity & location | `GenomicSequenceLocationCategory` | +| classification | `http://edamontology.org/topic_2885` | +| SNP alleles | `VariationSnpAlleleCategory` | +| Indel alleles | `VariationIndelAlleleCategory` | +| strain / call stats | `VariationStrainStatsCategory` | +| gene linkage, effect rollups, collapsed, both tables | `http://edamontology.org/topic_0199` | + +Hand-typing ~50 tab-delimited lines with 14 columns each is how tab-count bugs get made. +Generate them instead. Write this script to +`/tmp/claude-1000/-home-jbrestel-workspaces-agentic-veupath-dev/gen-variation-ontology.py` +— it names every attribute explicitly, so nothing is left to inference: + +```python +#!/usr/bin/env python3 +"""Emit individuals.txt lines for the variation record. 14 tab-separated columns.""" + +RC = "VariationRecordClasses.VariationRecordClass" + +LOC = ("GenomicSequenceLocationCategory", "GenomicSequenceLocationCategory") +POLY = ("http://edamontology.org/topic_2885", "DNA Polymorphism") +GENV = ("http://edamontology.org/topic_0199", "Genetic Variation") +SNPA = ("VariationSnpAlleleCategory", "SNP Alleles") +INDA = ("VariationIndelAlleleCategory", "Indel Alleles") +STATS = ("VariationStrainStatsCategory", "Strain Statistics") + +FULL = ["results", "record", "download"] +REC = ["", "record", "download"] +INT = ["", "record-internal", ""] + +# (parent, targetType, name, scopes) +ROWS = [ + # identity & location + (LOC, "attribute", "variation_location", FULL), + (LOC, "attribute", "sequence_source_id", FULL), + (LOC, "attribute", "location", FULL), + (LOC, "attribute", "location_text", INT), + (LOC, "attribute", "chromosome_order_num", INT), + (LOC, "attribute", "organism", FULL), + (LOC, "attribute", "organism_text", INT), + (LOC, "attribute", "formatted_organism", INT), + (LOC, "attribute", "ncbi_tax_id", REC), + (LOC, "attribute", "dataset", FULL), + # classification + (POLY, "attribute", "variant_type", FULL), + (POLY, "attribute", "is_coding", FULL), + (POLY, "attribute", "reference_strain", FULL), + # record overview + (POLY, "attribute", "record_overview", REC), +] + +for prefix, parent in (("snp", SNPA), ("indel", INDA)): + for suffix in ["ref_allele", "major_allele", "major_allele_frequency", + "major_allele_strain_count", "minor_allele", + "minor_allele_frequency", "minor_allele_strain_count", + "major_genomic_hgvs", "minor_genomic_hgvs"]: + ROWS.append((parent, "attribute", f"{prefix}_{suffix}", FULL)) + for suffix in ["major_allele_and_freq", "minor_allele_and_freq"]: + ROWS.append((parent, "attribute", f"{prefix}_{suffix}", REC)) +ROWS.append((INDA, "attribute", "indel_frame_effect", FULL)) + +for name in ["distinct_strain_count", "called_strain_count", "no_call_strain_count", + "call_rate", "total_ploidy_count", "het_strain_count", + "ref_allele_frequency"]: + ROWS.append((STATS, "attribute", name, FULL)) + +for name in ["gene_ids", "linkedGeneIds", "most_severe_impact_snpeff", + "most_severe_impact_product_call", "effect_summary_snpeff", + "effect_summary_product_call", "collapsed_allele", + "collapsed_minor_allele_frequency"]: + ROWS.append((GENV, "attribute", name, FULL)) +ROWS.append((GENV, "attribute", "gene_count", INT)) + +for name in ["TranscriptProducts", "PredictedEffects"]: + ROWS.append((GENV, "table", name, REC)) + +for (parent_iri, parent_label), target, name, scopes in ROWS: + cols = [f"{RC}.{name}", parent_iri, parent_label, RC, target, name, + "", "", "", "", ""] + scopes + assert len(cols) == 14, (name, len(cols)) + print("\t".join(cols)) +``` + +Run it and append the output: + +```bash +python3 /tmp/claude-1000/-home-jbrestel-workspaces-agentic-veupath-dev/gen-variation-ontology.py \ + >> ~/workspaces/plasmodb/ApiCommonModel/Model/lib/wdk/ontology/individuals.txt +``` + +Then confirm the count: + +```bash +grep -c '^VariationRecordClasses' ~/workspaces/plasmodb/ApiCommonModel/Model/lib/wdk/ontology/individuals.txt +``` + +Expected: `55`. (Verified by running the script — it emits 55 lines, every one with +exactly 14 tab-separated columns, enforced by the `assert` in the loop.) + +The three category-node lines from Step 2 are added by hand, not by this script — they +have a different shape (no record class, `category` target type). + +- [ ] **Step 4: Verify tab structure is consistent** + +```bash +cd ~/workspaces/plasmodb/ApiCommonModel && \ + awk -F'\t' '/^VariationRecordClasses/ {print NF}' Model/lib/wdk/ontology/individuals.txt | sort -u +``` + +Expected: a single value, matching what existing `SnpRecordClasses` lines produce: + +```bash +awk -F'\t' '/^SnpRecordClasses/ {print NF}' Model/lib/wdk/ontology/individuals.txt | sort -u +``` + +If the two differ, the tab count is wrong and the OWL build will fail or misplace nodes. + +- [ ] **Step 5: Build the ontology — do NOT also run `wb model`** + +```bash +cd ~/workspaces/agentic-veupath-dev && bin/veup-build.sh plasmodb wb ontology +``` + +Expected: `individuals.owl` → `categories_merged.owl` regenerated, model rebuilt, webapp +reloaded. A failure mentioning an unresolvable IRI means a parent category name is +misspelled. + +- [ ] **Step 6: Verify placement in the assembled tree** + +From an authenticated app page: + +```js +const o = await (await fetch('/plasmo.jbrestel/service/ontologies/Categories')).json(); +const hits = []; +(function walk(n, parent) { + const p = n.properties || {}; + if ((p.name || [])[0]?.startsWith?.('snp_minor_allele') || + (p.name || [])[0] === 'PredictedEffects' || + (p.name || [])[0] === 'distinct_strain_count') + hits.push({ name: p.name, parent: parent?.properties?.['EuPathDB alternative term'] }); + (n.children || []).forEach(c => walk(c, n)); +})(o.tree, null); +hits +``` + +Expected: `snp_minor_allele` under a parent displaying "SNP Alleles", +`distinct_strain_count` under "Strain Statistics", `PredictedEffects` under the Genetic +Variation node. + +**Caveat:** `/plasmo.jbrestel/service/ontologies/Categories` is **not** project-filtered. For +"does this site actually have the item", trust +`/plasmo.jbrestel/service/record-types/variation` instead. + +- [ ] **Step 7: Commit** + +```bash +cd ~/workspaces/plasmodb/ApiCommonModel +git add Model/lib/wdk/ontology/individuals.txt +git commit -m "Categorize variation record attributes and tables + +Adds SNP Alleles, Indel Alleles, and Strain Statistics category nodes so +the two allele classes read as distinct sections on the record page. + +Requires wb ontology; wb model alone does not regenerate the OWL and +would leave every attribute uncategorized with no error." +``` + +--- + +## Task 13: Final end-to-end verification + +**Files:** none + +- [ ] **Step 1: Confirm registration is complete** + +From an authenticated app page: + +```js +const rt = await (await fetch('/plasmo.jbrestel/service/record-types/variation')).json(); +({ attributes: rt.attributes.length, tables: rt.tables.map(t => t.name) }) +``` + +Expected: `tables` = `["TranscriptProducts", "PredictedEffects"]`, and `attributes` +covering everything from Tasks 4-9. + +- [ ] **Step 1a: Perform the visual checks deferred from Tasks 5-11** + +These could not run earlier: record pages render from the category ontology, which only +exists once Task 12 has run. Do them now, for real, rather than assuming earlier tasks +covered them. + +For each of the three variant-type records below, load the page and confirm with the DOM: + +```js +const ov = document.querySelector('.eupathdb-RecordOverview'); +({ overviewPresent: !!ov, + overviewText: ov ? ov.innerText.replace(/\n{2,}/g,'\n') : null, + sections: [...document.querySelectorAll('h2,h3,h4')].map(e => e.innerText.trim()).filter(Boolean), + bodyLen: document.body.innerText.length }) +``` + +Required: +- `overviewPresent` must be `true`. If it is `false`, the `record_overview` attribute has no + ontology entry with a `record` scope — check Task 12's generator emitted it. +- `Variant_Pf3D7_01_v3_29514` (SNV): SNP allele values shown, Indel blocks empty +- `Variant_Pf3D7_01_v3_18` (INDEL): Indel values shown, SNP blocks empty +- `Variant_Pf3D7_01_v3_12` (MIXED): **BOTH** blocks populated — the acceptance test for the + whole two-section design +- No literal unsubstituted `$$name$$` text on any page. Detect it without tripping tool + content filters by testing for the delimiter indirectly: + `document.body.innerText.split(String.fromCharCode(36,36)).length - 1` — expect `0`. +- Both record tables present and populated (5 codon rows and 2 effect rows for + `Variant_Pf3D7_01_v3_29514`). + +- [ ] **Step 2: Walk all three variant types with clean logs** + +```bash +cd ~/workspaces/agentic-veupath-dev && bin/veup-logs.sh plasmodb mark final +``` + +Load each in the browser, letting each settle: +- `.../plasmo.jbrestel/app/record/variation/Variant_Pf3D7_01_v3_29514` — SNV, coding, 5 codon rows, 2 effect rows +- `.../plasmo.jbrestel/app/record/variation/Variant_Pf3D7_01_v3_18` — INDEL, indel section only +- `.../plasmo.jbrestel/app/record/variation/Variant_Pf3D7_01_v3_12` — MIXED, **both** allele sections, intergenic effects + +```bash +cd ~/workspaces/agentic-veupath-dev && bin/veup-logs.sh plasmodb since final --quiet +``` + +Expected: error logs `silent:`. + +- [ ] **Step 3: Confirm the cross-project record classes still build** + +The record class is declared for 9 projects but only 3 have data. Confirm the other +projects' models still load — a project with no variation data must build cleanly and +simply return no records. + +Ask the user which other project instance to build against, if any is available. If none +is, note it explicitly as unverified rather than claiming cross-project correctness. + +- [ ] **Step 4: Report** + +Summarize: what was built, what was verified with what evidence, and anything left +unverified (notably Step 3, and the `linkedGeneIds` plain-text limitation from Task 8). + +--- + +## Task 14: Flip the stub back to the real tuning table + +**Do this once the central tuning job has built `apidbtuning.VariationAttributes`.** This +task is what stops a developer-schema reference from reaching a merge. + +**Files:** +- Modify: `Model/lib/wdk/model/records/variationAttributeQueries.xml` +- Modify: `Model/lib/wdk/model/records/variationTableQueries.xml` + +- [ ] **Step 1: Confirm the real table exists and matches the stub** + +```bash +psql -h localhost -p 5432 -d unidb_shu_a -c " +select (select count(*) from apidbtuning.variationattributes) as real_table, + (select count(*) from jbrestel.variationattributes) as stub" +``` + +Expected: equal counts. If `real_table` errors, the tuning job has not run — stop. + +If the counts differ, do not proceed on the assumption that the stub was right: compare a +sample before flipping, since a difference means the tuning-manager path produced +something the verified SQL did not. + +```bash +psql -h localhost -p 5432 -d unidb_shu_a -c " +select * from apidbtuning.variationattributes +except select * from jbrestel.variationattributes limit 5" +``` + +Expected: no rows. + +- [ ] **Step 2: Rewrite all six references** + +```bash +cd ~/workspaces/plasmodb/ApiCommonModel/Model/lib/wdk/model/records +sed -i 's/jbrestel\.VariationAttributes/ApidbTuning.VariationAttributes/g' \ + variationAttributeQueries.xml variationTableQueries.xml +``` + +- [ ] **Step 3: Remove the TEMPORARY STUB comment from both files** + +Delete the three-line `` block added in Task 3 Step 1a from +the top of `variationAttributeQueries.xml` and `variationTableQueries.xml`. + +- [ ] **Step 4: Verify no stub reference survives** + +```bash +cd ~/workspaces/plasmodb/ApiCommonModel && \ + grep -rn 'jbrestel' Model/lib/wdk/ Model/lib/xml/ ; echo "exit: $?" +``` + +Expected: no matches (`exit: 1` from grep). **Any match here is a blocker for merge.** + +```bash +cd ~/workspaces/plasmodb/ApiCommonModel/Model/lib/wdk/model/records && \ + grep -c 'ApidbTuning\.VariationAttributes' variationAttributeQueries.xml variationTableQueries.xml +``` + +Expected: `4` and `2`. + +- [ ] **Step 5: Rebuild and re-verify** + +```bash +cd ~/workspaces/agentic-veupath-dev && bin/veup-build.sh plasmodb wb model +``` + +Then re-run the Task 13 Step 2 walk of all three variant-type records, confirming the +pages are unchanged from the stubbed state. + +- [ ] **Step 6: Commit** + +```bash +cd ~/workspaces/plasmodb/ApiCommonModel +git add Model/lib/wdk/model/records/variationAttributeQueries.xml \ + Model/lib/wdk/model/records/variationTableQueries.xml +git commit -m "Point the variation record at the real tuning table + +Replaces the temporary jbrestel.VariationAttributes stub with +ApidbTuning.VariationAttributes now that the tuning job has built it. +Record pages verified unchanged across SNV, INDEL, and MIXED loci." +``` + +- [ ] **Step 7: Drop the stub table** (optional, and only after Step 5 passes) + +```bash +psql -h localhost -p 5432 -d unidb_shu_a -c "DROP TABLE jbrestel.VariationAttributes" +``` + +--- + +## Deliberately not in this plan + +- **Searches / questions** for the variation record — separate spec and plan. +- **Removing the dead `snp` model XML.** `snpRecords.xml` and friends still reference the + non-existent `apidbtuning.SnpAttributes`; removal has its own blast radius + (`recordParams.xml`, `spanQuestions.xml`, `SnpsBySpanLogic`). +- **A sanity-test file.** `apiCommonModel-sanity.xml` imports a directory that does not + exist; the whole sanity model is dead. Do not extend it. +- **Reconciling `downstream_of_frameshift_strain_ids`**, which is in the live table but + not the checked-in DDL and is 100% null. +- **Per-strain / VCF data** — spec §9. +- **A `webready.variationattributes_p` variant.** 38 of 54 tuning tables are now thin + copies of `webready.*_p` tables built by a separate pipeline, which appears to be the + direction of travel. This plan uses in-tuning-manager SQL (the pattern the other 16 + still follow) because it is self-contained and executable now. If the data team wants + the webready route, only Task 2 changes. diff --git a/docs/superpowers/specs/2026-07-30-variation-record-design.md b/docs/superpowers/specs/2026-07-30-variation-record-design.md new file mode 100644 index 0000000000..1ec6f35284 --- /dev/null +++ b/docs/superpowers/specs/2026-07-30-variation-record-design.md @@ -0,0 +1,518 @@ +# Variation record — design + +**Date:** 2026-07-30 +**Status:** approved (attributes and record-page tables) +**Scope:** WDK record class, attributes, tuning table, and record-page tables for a new +`variation` record. **Searches (questions) are out of scope** — a separate spec. +**Implementation target:** `ApiCommonModel` (branch `dnaseq-merge-experiments`) + +## 1. Purpose + +Introduce a `variation` record representing a **single variant locus** — one +(sequence, position) pair — uniquely identified by `source_id`. It replaces the +deprecated `snp` record, which is not merely unused but **unbuildable**: its sole +backing relation `apidbtuning.SnpAttributes` does not exist in the current build, and +the per-strain table it depended on (`apidb.SequenceVariation`) has 0 rows. + +The new record is deliberately **aggregate-only**. Per-strain allele data is too large +for a relational table and lives in VCF files; see §9 for how the record is shaped to +accept it later without being reshaped. + +## 2. Data sources + +Base tables from +`ApidbSchema/Main/lib/sql/apidbschema/Postgres/createVariationTables.sql`. +Row counts and cardinalities below were measured against `unidb_shu_a` on 2026-07-30. + +### `apidb.VariationFeature` — 4,390,908 rows +One row per locus. PK `(sequence_source_id, location)`, `UNIQUE (source_id)`. +Already record-shaped, which is what makes the hybrid sourcing rule in §3.1 viable. + +`source_id` format: `Variant__` (e.g. +`Variant_Pf3D7_01_v3_29514`). + +Carries **two parallel allele summaries** — 9 `snp_*` columns and 10 `indel_*` columns +(19 in total; `indel_frame_effect` has no SNP counterpart) — plus locus-level call +statistics. + +| `variant_type` | `is_coding=0` | `is_coding=1` | populates | +|---|---|---|---| +| `SNV` | 2,350,257 | 1,649,759 | `snp_*` only | +| `INDEL` | 232,957 | 28,085 | `indel_*` only | +| `MIXED` | 116,786 | 13,064 | **both** | + +The 129,850 `MIXED` loci describe the same strains two ways. Example +`Pf3D7_01_v3:12` — SNP view: major `A`, minor `C`; indel view: major `A`, minor `AC`. +This is why §6 keeps two named sections rather than collapsing. + +Observed data note: `snp_major_genomic_hgvs` / `indel_major_genomic_hgvs` are null +whenever the major allele equals the reference, which was true at every sampled locus. +The `*_minor_genomic_hgvs` columns are the ones that reliably carry content. + +### `apidb.VariationTranscriptProduct` — 4,595,009 rows +Locus × transcript × observed codon. `na_feature_id` resolves exclusively to +`subclass_view = 'Transcript'` (23,456 distinct transcripts). + +- 1,690,908 loci have transcript products; per locus: 1 transcript (1,687,832), + 2 (3,024), 3 (52). +- By **gene**: 1 gene (1,690,354), 2 genes (554). +- Every `is_coding = 1` locus has at least one transcript product — no orphans. + +### `apidb.VariationEffect` — 6,789,700 rows +Locus × allele × transcript × caller. `na_feature_id` is nullable (intergenic). +48 distinct `(impact, effect, source)` combinations. + +Two callers, which are **complementary rather than redundant**: + +| | rows | +|---|---| +| `snpeff` | 4,849,915 | +| `product_call` | 1,939,785 | +| paired and agreeing on `effect` | 1,574,445 | +| paired and **disagreeing** | 377,979 (19% of paired) | +| `product_call` with no `snpeff` counterpart | 139 | + +Pairing is on `(sequence_source_id, location, allele, na_feature_id)`, so rows with a +null `na_feature_id` (intergenic) are excluded from the pairing counts — the +agree/disagree figures describe gene-associated calls only. + +`snpeff` annotates each variant in isolation. `product_call` is strain-aware and knows +when a variant sits downstream of a frameshift. Top disagreements: + +| `product_call` | `snpeff` | rows | +|---|---|---| +| `downstream_frameshift` | `synonymous_variant` | 107,784 | +| `downstream_frameshift` | `missense_variant` | 104,550 | +| `missense_variant` | `synonymous_variant` | 60,020 | +| `synonymous_variant` | `missense_variant` | 22,953 | +| `downstream_frameshift` | `frameshift_variant` | 13,033 | +| `inframe_deletion_unnormalized` | `disruptive_inframe_deletion` | 10,586 | + +By gene via `VariationEffect` (broader than products — covers intron/UTR/non-coding): +1 gene (2,853,792), 2 genes (25,042), 3 genes (503). + +### Provenance +`external_database_release_id` → 3 loaded datasets: `pfal3D7_dnaSeqVariations` +(620,467), `tbruTREU927_dnaSeqVariations` (652,689), `afumAf293_dnaSeqVariations` +(3,117,752). + +### Schema drift to flag +The live `apidb.VariationTranscriptProduct` has a column **absent from the checked-in +DDL**: `downstream_of_frameshift_strain_ids`. It is 100% null (0 of 4,595,009 rows). +This design ignores it. Either the DDL or the table should be reconciled; if the +column is intended to carry per-strain IDs it overlaps the §9 VCF work and should be +designed there, not smuggled in as a string column. + +## 3. Design decisions + +### 3.1 Sourcing rule (governing invariant) + +> **Intrinsic per-locus facts read directly from `apidb.VariationFeature`. Anything +> derived, aggregated, or requiring a join lives in `apidbtuning.VariationAttributes`.** + +Rationale: three options were considered — a full BFMV-style denormalized tuning table +(as the old `snp` record used), direct queries against base tables only, and this +hybrid. Direct-only is disqualified by WDK's attribute-query contract, which is +"one query returning every record's value for these columns" and therefore structurally +wants a pre-materialized one-row-per-record relation; the derived rollups would be +recomputed on every query over a 4.4M × 6.8M join. + +A hybrid is normally the worst option, because it leaves attributes scattered with no +rule for where a new column belongs. It is sound **here specifically** because +`VariationFeature` is already one row per locus with a unique `source_id`, so the rule +above is decidable for any future column with no exceptions. + +The hybrid also wins on one concrete axis: it keeps large derived strings out of the +tuning table entirely (see §3.4). + +### 3.2 Both allele classes are preserved + +Two named attribute sections, "SNP Alleles" and "Indel Alleles". No collapse, because +collapsing loses half the data for 129,850 `MIXED` loci and buries the choice in a +`COALESCE` no one can see from the page. + +**Correction (verified 2026-07-30): the sections are NOT hidden when their class is absent.** +An earlier draft of this spec claimed they would be. WDK renders a null attribute inside a +record-page section as a "No data available" row, and there is no declarative way to hide a +whole section — so a pure-SNV locus shows a 12-row empty "Indel Alleles" section (~4.0M +loci) and an INDEL locus shows an empty "SNP Alleles" one (~261k loci). + +The load-bearing half of this decision holds: both classes are preserved and never +collapsed. Only the cosmetic half is unmet. Note the contrast with the record **overview**, +which is a `
` where the client *does* drop pairs whose value is empty — which is why +§6.7's `*_allele_and_freq` strings are assembled in SQL to be NULL rather than " ()". + +**Open decision for the record owner:** accept the empty sections, or hide them with a +client-side component. Deliberately left open rather than silently diverging. + +Because WDK tables render **only on individual record pages**, results-page columns must +be pre-aggregated. Two collapsed columns exist for that purpose only +(`collapsed_allele`, `collapsed_minor_allele_frequency`, §6.7). The two sections remain +canonical; the collapsed columns are explicitly derived conveniences. + +### 3.3 Gene linkage is many-to-many + +The tables are the source of truth. Any single-gene attribute is an **aggregate**, never +a lookup. 99.1% of loci are single-gene, but 25,545 loci are not, and a `string_agg` +today costs nothing while a baked-in single-gene assumption would need unpicking from +the tuning table, the summary columns, the download reporter, and any future search +param. + +Consequence: **no record-level `gene_strand`**, and no gene-strand reverse-complemented +flank attributes. Strand is not singular once a locus can hit two genes, so strand moves +into the transcript table (§7.1) where it is unambiguous. This is the one thing the old +`snp` record had that is deliberately dropped. + +### 3.3a Tuning-table dependency ordering + +`VariationAttributes` depends on two other tuning tables — `TranscriptAttributes` (for +the gene aggregate) and `GenomicSeqAttributes` (for `project_id`/`organism`). The tuning +manager expresses this natively: + +```xml + + + + + + +``` + +This matters because built out of order the gene aggregate yields a silently empty +`gene_ids` rather than an error. Declaring both `internalDependency` elements is not +optional. + +### 3.4 No sequence-context attributes + +The old `snp` record had `lflank`, `rflank`, `snp_context`, and gene-strand variants of +each, computed by `substr` against `dots.NaSequence`. These are **not carried forward** — +no longer needed; JBrowse covers genomic context. + +This removes the record's only dependency on `dots.NaSequence` at attribute-query time, +and avoids materializing ~570 MB of flank strings across 4.4M rows to serve attributes +that would only ever render on a single record page. + +### 3.5 Effect rollups are per-caller + +`most_severe_impact` and `effect_summary` are each **two columns**, one per caller. +Merging them (worst-case-wins across both) would produce a single sortable column, but +the 19% disagreement rate is the scientific content of the pipeline, not noise to be +resolved — a user must be able to see that `product_call` says `downstream_frameshift` +where `snpeff` says `synonymous_variant`. A merged impact alongside split effect lists +would also be an odd asymmetry. + +## 4. Record class + +``` +recordClassSet name="VariationRecordClasses" + includeProjects="AmoebaDB,CryptoDB,FungiDB,MicrosporidiaDB,PiroplasmaDB, + PlasmoDB,TriTrypDB,ToxoDB,UniDB" +``` +That is the deprecated `SnpRecordClass` project list verbatim — every project that had +`snp` records gets `variation` records. (Data is currently loaded for only PlasmoDB, +TriTrypDB, and FungiDB; the record simply returns nothing for the others until loaded.) + +``` +recordClass name="VariationRecordClass" urlName="variation" + displayName="Variation" displayNamePlural="Variations" +``` + +**Primary key:** `source_id` + `project_id`, with `project_id` excluded for UniDB — +matching `SnpRecordClass`. `aliasQueryRef` points at `VariationAttributes.VariationAlias`. + +**`idAttribute`** `primary_key`, display `$$source_id$$`. + +**Coordinate invariant (load-bearing):** `sequence_source_id` and `location` are +exposed, non-internal, independently addressable attributes — never merely parsed out of +`source_id`. This is the coordinate a tabix/VCF lookup and an EDA join both key on, and +it is what makes §9 additive rather than a reshape. + +**Reporters:** `attributesTabular`, `tableTabular`, `fullRecord`, `xml`, `json` — same +set and configuration as `SnpRecordClass`. + +**`testParamValues`** (coding SNVs with transcript products, verified present): + +| project | `source_id` | +|---|---| +| PlasmoDB | `Variant_Pf3D7_01_v3_100057` | +| TriTrypDB | `Variant_11L3_v3_26886` | +| FungiDB | `Variant_Chr1_A_fumigatus_Af293_1000005` | +| UniDB | `Variant_Pf3D7_01_v3_100057` (no `project_id`) | + +Remaining projects need values once their data loads; until then their record class has +no testable ID. Note this rather than inventing IDs. + +## 5. `apidbtuning.VariationAttributes` + +One row per locus (4,390,908). Derived, aggregated, or join-requiring columns **only** — +per §3.1. + +| column | source / derivation | +|---|---| +| `source_id` | `VariationFeature.source_id` (PK half) | +| `project_id` | taxon → project mapping (PK half) | +| `sequence_source_id`, `location` | join key to base tables; §4 invariant | +| `location_text` | `to_char(location,'99,999,999')` | +| `organism`, `ncbi_tax_id` | join `apidbtuning.GenomicSeqAttributes` on `source_id = sequence_source_id` | +| `chromosome_order_num` | same join; default sort key | +| `dataset` | `coalesce(DatasetPresenter.display_name, ExternalDatabase.name)` — see note | +| `gene_ids` | `string_agg(DISTINCT gene_source_id)` over both child tables | +| `gene_count` | `count(DISTINCT gene_source_id)` | +| `most_severe_impact_snpeff` | rank aggregate, `source='snpeff'` | +| `most_severe_impact_product_call` | rank aggregate, `source='product_call'` | +| `effect_summary_snpeff` | `string_agg(DISTINCT effect)`, `source='snpeff'` | +| `effect_summary_product_call` | `string_agg(DISTINCT effect)`, `source='product_call'` | +| `collapsed_allele` | §6.7 | +| `collapsed_minor_allele_frequency` | §6.7 | + +**Impact rank:** `HIGH` > `MODERATE` > `LOW` > `MODIFIER`. Loci with no +`VariationEffect` row for a given caller get null for that caller's two columns (common +for `product_call`, which covers 2.9M fewer loci than `snpeff`). + +**Gene aggregate:** built from `VariationEffect` **and** +`VariationTranscriptProduct`, joined to `apidbtuning.TranscriptAttributes` on +`na_feature_id`. `VariationEffect` alone reaches 2.88M loci vs 1.69M for products, so +`VariationEffect` supplies most of the coverage; products are unioned in so a locus with +a product but no effect row is not missed. + +**`project_id` / `organism` / `chromosome_order_num` derivation:** all three come from +`apidbtuning.GenomicSeqAttributes` joined on `source_id = sequence_source_id`. Verified +to match **100%** of the 4,390,908 loci (3 projects, 3 organisms). This is a cleaner +source than `dots.NaSequence` → taxon and adds an `internalDependency` on +`GenomicSeqAttributes`. + +**`dataset` caveat.** The three loaded external databases +(`pfal3D7_dnaSeqVariations` etc.) have **no matching `apidbtuning.DatasetPresenter` +row**, so `dataset` renders the raw external-database name today. This is not a model +bug: dataset presenters exist per **dnaseq experiment** +(`*_dnaseqExperiment_RSRC`, one per strain/isolate collection), whereas a +`*_dnaSeqVariations` external database is the *merged* call set across many such +experiments. So `dataset` is a single provenance string ("which call set"), and the list +of contributing experiments is inherently per-strain and therefore belongs to the §9 +seam, not here. The `coalesce` is retained so a future presenter is picked up +automatically. + +Indexes: PK on `(source_id, project_id)`, plus `(sequence_source_id, location)` and +`(chromosome_order_num, location)` for the default sort. + +Everything **not** in this table — `variant_type`, `is_coding`, all 19 +`snp_*`/`indel_*` columns, `reference_strain`, `call_rate`, and every strain count — +reads directly from `apidb.VariationFeature`. + +## 6. Attributes + +Source key: **VF** = `apidb.VariationFeature` (direct attribute query) · +**VA** = `apidbtuning.VariationAttributes` · **text** = WDK `textAttribute` + +### 6.1 Identity & location +| attribute | src | display name | notes | +|---|---|---|---| +| `primary_key` | — | Variation ID | `idAttribute` | +| `sequence_source_id` | VA | Sequence | | +| `location` | VA | Position | numeric, sortable | +| `location_text` | VA | — | comma-formatted | +| `variation_location` | text | Location | `$$sequence_source_id$$: $$location_text$$` | +| `chromosome_order_num` | VA | Chromosome | default sort | +| `organism_text` | VA | Organism | download form | +| `formatted_organism` | VA | Organism | italic abbreviated form, as `snp` | +| `organism` | text | Organism | italic wrapper over `organism_text` | +| `ncbi_tax_id` | VA | NCBI Taxon ID | `inReportMaker="false"` | +| `dataset` | VA | Variant Call Set | named for what it is: the merged call set, not a presenter-backed dataset | + +### 6.2 Classification +| attribute | src | notes | +|---|---|---| +| `variant_type` | VF | `SNV` / `INDEL` / `MIXED` | +| `is_coding` | VF | rendered `coding` / `non-coding` | +| `reference_strain` | VF | | + +### 6.3 SNP alleles — section "SNP Alleles", hidden when `snp_ref_allele is null` +All VF: `snp_ref_allele`, `snp_major_allele`, `snp_major_allele_frequency`, +`snp_major_allele_strain_count`, `snp_minor_allele`, `snp_minor_allele_frequency`, +`snp_minor_allele_strain_count`, `snp_major_genomic_hgvs`, `snp_minor_genomic_hgvs`. + +Plus text attributes `snp_major_allele_and_freq` / `snp_minor_allele_and_freq` +rendering `C (0.9571)`. + +`help` on both `*_genomic_hgvs` attributes must state that the major-allele HGVS is null +when the major allele equals the reference, so it does not read as missing data. + +### 6.4 Indel alleles — section "Indel Alleles", hidden when `indel_ref_allele is null` +The same nine shapes with the `indel_` prefix, plus `indel_frame_effect`, plus the two +matching `*_allele_and_freq` text attributes. Same HGVS `help`. + +### 6.5 Strain / call statistics +All VF: `distinct_strain_count`, `called_strain_count`, `no_call_strain_count`, +`call_rate`, `total_ploidy_count`, `het_strain_count`, `ref_allele_frequency`. + +This group is the natural anchor for the future strain table (§9). + +**Semantics caveat, measured 2026-07-30.** These counts do **not** partition one +population, despite the naming inviting that reading. `called_strain_count + +no_call_strain_count` is near-constant per dataset (232 for the Pf panel, 4 for another) — +it is the size of the whole assayed strain panel. `distinct_strain_count` tracks something +narrower, roughly `total_ploidy_count / 2`, i.e. strains actually carrying a called allele +configuration. At `Variant_Pf3D7_01_v3_29514`: 159 + 57 = 216, while +`distinct_strain_count` and `total_ploidy_count` are both 160. + +So "Strain Count" and "Called Strain Count" come from different universes. This is a +property of `apidb.VariationFeature`, not of this record. It matters for §9: whoever builds +the per-strain table should reconcile the naming, or at minimum document which denominator +each frequency column uses, before users start computing ratios across the two. + +All seven columns are schema-nullable, though no NULLs exist in the current 4,390,908 rows. + +### 6.6 Gene linkage +| attribute | src | notes | +|---|---|---| +| `gene_ids` | VA | aggregated, sortable as string; used directly in summaries and the overview | +| `gene_count` | VA | | + +**Correction (2026-07-30): there is no `linkedGeneIds` attribute.** This spec originally +called for "one link per ID", but WDK's `linkAttribute` builds exactly one URL and cannot +render N links from an aggregated string. A `textAttribute` fallback rendered text +byte-identical to `gene_ids`, i.e. a second column also displayed as "Gene ID(s)" with the +same content, so it was removed. Per-ID hyperlinks would need a client-side component; +until then `gene_ids` is plain text. + +### 6.7 Effect rollups & collapsed summary columns +All VA: `most_severe_impact_snpeff`, `most_severe_impact_product_call`, +`effect_summary_snpeff`, `effect_summary_product_call`. + +`collapsed_allele` — `ref>minor` per class, both classes joined with `; ` for `MIXED`: + +| `variant_type` | example | +|---|---| +| `SNV` | `T>C` | +| `INDEL` | `AC>A` | +| `MIXED` | `A>C; A>AC` | + +`collapsed_minor_allele_frequency` — `greatest()` of the two classes' minor-allele +frequencies. Exists because MAF is the most-sorted-on quantity in any variant table and +a `MIXED` locus otherwise has no single sortable frequency. + +**Impact columns need a rank sort key.** `most_severe_impact_*` hold text, so WDK sorts +them alphabetically (`HIGH, LOW, MODERATE, MODIFIER`) — contradicting the documented +severity order. Each therefore declares `sortingColumn` pointing at a computed rank column +(`CASE HIGH→4, MODERATE→3, LOW→2, MODIFIER→1`) emitted by the same attribute query. The +rank columns are `internal="true"`. No tuning-table column is needed for this. + +Both carry `help` naming them as derived, with the two sections as canonical. + +### 6.8 Record overview + +Two-panel `textAttribute` `record_overview`, following `SnpRecordClass`'s structure: + +- **Left** — Organism, Location, Variant Type, Coding?, Reference Strain, Gene(s) + (`linkedGeneIds`), Most Severe Impact (both callers). +- **Right** — the SNP Alleles and Indel Alleles sections (§6.3, §6.4), then call + statistics (`distinct_strain_count`, `called_strain_count`, `no_call_strain_count`, + `call_rate`, `het_strain_count`). + +### 6.9 Default summary & sorting +``` +summary = variation_location, linkedGeneIds, variant_type, collapsed_allele, + collapsed_minor_allele_frequency, most_severe_impact_snpeff, + most_severe_impact_product_call, distinct_strain_count +sorting = chromosome_order_num asc, location asc +``` +Plus an `organism`-leading variant for UniDB / EuPathDB, mirroring `SnpRecordClass`. + +## 7. Record-page tables + +> Tables render **only** on individual record pages, so everything here is scoped to +> one locus. + +### 7.1 `TranscriptProducts` — "Variant Products by Transcript" +`queryRef="VariationTables.TranscriptProducts"`. +`apidb.VariationTranscriptProduct` joined to `apidbtuning.TranscriptAttributes` on +`na_feature_id`. One row per (transcript, observed codon) — 5 rows for +`Variant_Pf3D7_01_v3_29514`. + +Columns: `linkedGeneId`, `linkedTranscriptId`, `gene_product`, `strand` (the +unambiguous home for strand, per §3.3), `pos_in_cds`, `pos_in_protein`, `codon`, +`pos_in_codon`, `product`, `matches_ref_codon`, `matches_ref_product`, `strain_count`, +`hgvs_p`. + +### 7.2 `PredictedEffects` — "Predicted Effects" +`queryRef="VariationTables.PredictedEffects"`. +`apidb.VariationEffect` **left** joined to `apidbtuning.TranscriptAttributes` (left, +because `na_feature_id` is null for intergenic calls). + +Columns: `allele`, `linkedGeneId`, `transcript_source_id`, `impact`, `effect`, `hgvs_c`, +`source`. + +Note: `transcript_source_id` is plain text here, not a link. There is no standalone +transcript record to link to — `TranscriptProducts` (§7.1) anchors into the gene page +instead, and repeating that indirection on every effect row was not worth it. + +One table with a visible `source` column, not two tables — provenance travels with +every row, and the disagreement stays legible instead of requiring the user to read two +tables and diff them. `help` on `source` must explain that `snpeff` annotates variants +in isolation while `product_call` is strain-aware, and specifically what +`downstream_frameshift` means, since that accounts for 232,946 of the 377,979 +disagreements (62%). + +### 7.3 Deliberately absent +- **Strains / Samples** — deferred to §9. +- **Allele Summary**, **Country Summary** — both read `apidb.SequenceVariation` + (0 rows). They return with §9, sourced from VCF + EDA metadata, not rebuilt now. +- **Other variants at this location** — the old `Providers` table read + `apidbtuning.SnpChipAttributes`, which does not exist in this build. Out of scope. + +## 8. Category ontology + +New entries in `Model/lib/wdk/ontology/individuals.txt` for +`VariationRecordClasses.VariationRecordClass`. Because these are categorization changes, +**`wb ontology` is required** — `wb model` alone will not regenerate the OWL. + +Parent categories, reusing existing nodes: + +| group | parent | +|---|---| +| identity & location (§6.1) | `GenomicSequenceLocationCategory` | +| classification (§6.2) | `http://edamontology.org/topic_2885` (DNA Polymorphism) | +| SNP Alleles (§6.3) | new child node under `topic_2885`, display term "SNP Alleles" | +| Indel Alleles (§6.4) | new child node under `topic_2885`, display term "Indel Alleles" | +| strain / call statistics (§6.5) | new child node under `topic_2885`, "Strain Statistics" | +| gene linkage (§6.6) | `http://edamontology.org/topic_0199` (Genetic Variation) | +| effect rollups (§6.7) | `topic_0199` | +| both tables (§7) | `topic_0199` | + +Verify placement afterwards via `/service/ontologies/Categories` and +`/service/record-types/variation` from an authenticated app page — the assembled tree, +not the record page, is the source of truth. Note that `/service/ontologies/Categories` +is **not** project-filtered, so for "does this site actually have it" trust +`/service/record-types/variation`. + +## 9. Deferred: per-strain data (the seam) + +Per-strain allele data is too large for a relational table and lives in **VCF files**. +The longer-term goal is to read the VCF to populate strain/sample tables, joined with +sample metadata from **EDA**. + +This is not a future ``. It will be a service- or plugin-backed table, which +is a different *kind* of thing from every other table on this record. The seam is +therefore about **identity, not a query stub**: + +1. `sequence_source_id` and `location` are exposed first-class attributes (§4) — the + coordinate a tabix lookup and an EDA sample join both key on. Nothing has to + string-parse `source_id` to get coordinates. +2. `apidbtuning.VariationAttributes` guarantees exactly one row per locus keyed by + those coordinates — a single stable join target. +3. §6.5 groups the aggregate call statistics together, so the strain table lands + beside the counts it details rather than in an unrelated section. + +Get these right and the strain section is additive. The three tables removed in §7.3 +(Strains/Samples, Allele Summary, Country Summary) all return through this seam. + +## 10. Out of scope + +- **Searches / questions** — separate spec. +- **Deleting the `snp` record XML.** `snpRecords.xml`, `snpAttributeQueries.xml`, + `snpTableQueries.xml`, `snpQuestions.xml`, and their `individuals.txt` entries still + reference `apidbtuning.SnpAttributes`, which no longer exists. Removing them is a + separate change with its own blast radius (`recordParams.xml`, `spanQuestions.xml`, + and the `SnpsBySpanLogic` dynamic-span record all reference it). +- **JBrowse track integration** for the new record. +- Reconciling the `downstream_of_frameshift_strain_ids` drift noted in §2. From 6be5539aeae92c8e30a60b6a1443ebafd9b69573 Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Wed, 5 Aug 2026 09:41:48 -0400 Subject: [PATCH 23/98] Reconcile the variation plan's progress with what actually shipped MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The progress block had been stale since Task 4, claiming Task 5 was next while tasks 5-14 were all committed on this branch, and all 91 step checkboxes were unchecked. Replaced with a task-to-commit map covering 0-14, and checked the boxes. Also retires the stub warning. Both halves are settled: Task 14 (2723cda) flipped all six query references to ApidbTuning.VariationAttributes, and a tuning run built the real table on unidb_shu_a (variationattributes1121, 1688 MB, 4,390,908 rows, plus the view; all 17 va.* columns the model reads resolve). jbrestel.VariationAttributes no longer exists in the database. The plan's original assumption that the flip would cost dev-instance buildability no longer holds, and said so misleadingly. Task bodies keep their jbrestel snippets — that is what was executed — with a banner not to copy them forward. Co-Authored-By: Claude Opus 5 --- .../plans/2026-07-30-variation-record.md | 256 ++++++++++-------- 1 file changed, 142 insertions(+), 114 deletions(-) diff --git a/docs/superpowers/plans/2026-07-30-variation-record.md b/docs/superpowers/plans/2026-07-30-variation-record.md index 9e34710668..229ee4cbc1 100644 --- a/docs/superpowers/plans/2026-07-30-variation-record.md +++ b/docs/superpowers/plans/2026-07-30-variation-record.md @@ -14,37 +14,65 @@ > ### Progress > -> **Tasks 0-4 are done. Task 5 is next.** +> **All tasks (0-14) are complete.** Reconciled against the commit log on +> `dnaseq-merge-experiments` on 2026-08-05; the block below had been stale since Task 4. > -> Task 4 committed as `9bcf147` in `ApiCommonModel`: the record builds green, 13 -> attributes are registered, and -> `/plasmo.jbrestel/app/record/variation/Variant_Pf3D7_01_v3_29514` renders as -> "Variation: Variant_Pf3D7_01_v3_29514" with every service call 200 and all error logs -> silent. Building it surfaced three defects in this plan, now fixed: the app/service base -> URLs both include `/plasmo.jbrestel`, an empty `querySet` is invalid so +> | task | commit(s) | +> |---|---| +> | 0-1 preconditions, psql SELECT check | verification only, no commit | +> | 2 tuning table definition | `9ff6094` | +> | 3 build and verify the tuning table | see the note below — superseded by a tuning run | +> | 4 minimal buildable record | `9bcf147` | +> | 5 classification attributes | `48f13bc` | +> | 6 SNP and Indel allele sections | `d35e557`, `51cb41b` (assemble allele strings in SQL) | +> | 7 strain and call statistics | `af9b2aa`, `e0bb106` (rename to "Called Strain Count") | +> | 8 gene linkage, effect rollups, collapsed columns | `d8ccd76`, `e3571e2` (impact sort, MAF help) | +> | 9 record overview and default summary | `f9264c0`, `d636b44` (label allele rows by class) | +> | 10 TranscriptProducts table | `dd4efa6`, `9cc505b` (strain_count help text) | +> | 11 PredictedEffects table | `b19c82d` | +> | 12 category ontology placement | `cf0dd13`, `19b6527` (ontology parenting) | +> | 13 final end-to-end verification | `19b6527` — the four review findings it fixes are that pass's output; there is no separate report doc | +> | 14 flip the stub to the real tuning table | `2723cda` | +> +> Task 4's original detail, kept because the defects it surfaced are still worth knowing: +> the record built green with 13 attributes registered and +> `/plasmo.jbrestel/app/record/variation/Variant_Pf3D7_01_v3_29514` rendering with every +> service call 200 and all error logs silent. Three plan defects came out of it — the +> app/service base URLs both include `/plasmo.jbrestel`, an empty `querySet` is invalid so > `variationTableQueries.xml` moved to Task 10, and the snp imports sit inside a comment > block that would have swallowed the new imports. > -> **Tasks 0-3 detail.** +> Tuning table as verified at Task 3: **4,390,908 rows**, exactly matching +> `apidb.VariationFeature`, built in 66s with three indexes and `GRANT SELECT TO gus_r`. +> `gene_ids` populated for 2,879,337 loci, `most_severe_impact_snpeff` for 4,390,895, +> `most_severe_impact_product_call` for 1,690,908, `collapsed_allele` for all 4,390,908, +> 3 projects, **25,545 multi-gene loci** — matching the spec's figure exactly. All three +> spot-check loci correct, including `A>C; A>AC` for the MIXED locus. +> +> ### ✓ The developer-schema stub is gone (was: the model reads a stub) +> +> Resolved. This warning described Tasks 4-13 writing `jbrestel.VariationAttributes` into +> the query XML in six places, because `tuningManager` was not installed on the dev +> instance when the plan was written. > -> - Tuning definition committed to `ApiCommonModel` on `dnaseq-merge-experiments` as -> `9ff6094` (`apiTuningManager.xml`, +103 lines, XML validated, tuning-table count -> 54 → 55). -> - `jbrestel.VariationAttributes` built and verified: **4,390,908 rows** (exactly -> matching `apidb.VariationFeature`) in 66s, three indexes, `GRANT SELECT TO gus_r`. -> `gene_ids` populated for 2,879,337 loci, `most_severe_impact_snpeff` for 4,390,895, -> `most_severe_impact_product_call` for 1,690,908, `collapsed_allele` for all -> 4,390,908, 3 projects, **25,545 multi-gene loci** — matching the spec's figure -> exactly. All three spot-check loci correct, including `A>C; A>AC` for the MIXED locus. +> Both halves have since been settled, verified 2026-08-05: > -> ### ⚠ The model reads a developer-schema stub +> - **Model:** Task 14 (`2723cda`) flipped all six references to +> `ApidbTuning.VariationAttributes`. No `jbrestel` reference remains anywhere in the +> variation model XML. +> - **Database:** a tuning run (Jenkins) built the real thing on `unidb_shu_a` — +> `apidbtuning.variationattributes1121` (1688 MB, 4,390,908 rows) plus the +> `apidbtuning.variationattributes` view over it, `SELECT` granted to `gus_r`. All 17 +> `va.*` columns the model reads resolve against that view. `jbrestel.VariationAttributes` +> no longer exists. > -> `tuningManager` is **not installed** on this dev instance, so the table lives in -> `jbrestel.VariationAttributes` rather than `apidbtuning`. Tasks 4-13 therefore write -> `jbrestel.VariationAttributes` into the query XML in **six places**. +> So the merge blocker is clear, and the flip does **not** cost buildability on the dev +> instance the way this plan originally assumed it would. > -> **This must not reach a merge.** Task 14 flips it back and fails loudly if any stub -> reference survives. Do not skip it. +> The Task 4-11 bodies below still show `jbrestel.VariationAttributes` in their SQL +> snippets, deliberately: that is what was executed at the time, and rewriting them would +> describe a history that never happened. **Do not copy those snippets forward** — the +> committed XML is the current truth. Read them as a record, not as instructions. **Two repos are involved:** @@ -161,7 +189,7 @@ is the only one that grows large; keeping queries out of it is what keeps it rea **Files:** none (verification only) -- [ ] **Step 1: Confirm the ApiCommonModel branch and clean tree** +- [x] **Step 1: Confirm the ApiCommonModel branch and clean tree** ```bash cd ~/workspaces/plasmodb/ApiCommonModel && git branch --show-current && git status --porcelain @@ -171,7 +199,7 @@ Expected: `dnaseq-merge-experiments`, and no output from `git status` (clean tre branch differs, stop and ask — do not switch branches, because a switch requires `bin/veup-git-sync.sh plasmodb` afterwards and may not be what the user wants. -- [ ] **Step 2: Confirm the base tables exist and are populated** +- [x] **Step 2: Confirm the base tables exist and are populated** ```bash psql -h localhost -p 5432 -d unidb_shu_a \ @@ -183,7 +211,7 @@ psql -h localhost -p 5432 -d unidb_shu_a \ Expected: `4390908`, `4595009`, `6789700`. Small drift is fine (data may have been reloaded); if a count is `0`, stop — nothing downstream will work. -- [ ] **Step 3: Confirm the instance's appDb is the database you are querying** +- [x] **Step 3: Confirm the instance's appDb is the database you are querying** ```bash ssh cedar 'grep -A3 -i "appdb" /var/www/jbrestel.plasmodb.org/gus_home/config/model-config.xml | head -20' @@ -193,7 +221,7 @@ Expected: a `connectionUrl` naming `unidb_shu_a`. **If it names a different data stop and ask the user** — every psql assertion in this plan would be checking a database the site does not read, and `wb model` would fail on missing tables. -- [ ] **Step 4: Confirm the prerequisite tuning tables exist** +- [x] **Step 4: Confirm the prerequisite tuning tables exist** ```bash psql -h localhost -p 5432 -d unidb_shu_a \ @@ -213,7 +241,7 @@ below were chosen to cover all three `variant_type` values. **Files:** - Create: `/tmp/claude-1000/-home-jbrestel-workspaces-agentic-veupath-dev/scratch-va-spot.sql` (scratch, not committed) -- [ ] **Step 1: Write the scoped spot-check query** +- [x] **Step 1: Write the scoped spot-check query** Create the scratch file with this content. The base-table scans are filtered to one sequence so it returns in seconds rather than minutes; the unfiltered form is what goes @@ -287,7 +315,7 @@ WHERE v.sequence_source_id = 'Pf3D7_01_v3' AND v.location IN (12, 18, 29514) ORDER BY v.location ``` -- [ ] **Step 2: Run it and assert the expected values** +- [x] **Step 2: Run it and assert the expected values** ```bash psql -h localhost -p 5432 -d unidb_shu_a -x \ @@ -316,7 +344,7 @@ it is the case a collapsed single-allele column would have silently destroyed. because presenters exist per dnaseq *experiment* while `*_dnaSeqVariations` is the merged call set across many experiments. Do not "fix" this. -- [ ] **Step 3: Verify the GenomicSeqAttributes join loses no loci** +- [x] **Step 3: Verify the GenomicSeqAttributes join loses no loci** ```bash psql -h localhost -p 5432 -d unidb_shu_a -c " @@ -328,7 +356,7 @@ left join apidbtuning.genomicseqattributes g on g.source_id = v.sequence_source_ Expected: `total` = `matched` = `4390908`. An inner join is used in the tuning SQL, so a mismatch would silently drop loci. -- [ ] **Step 4: No commit** +- [x] **Step 4: No commit** Scratch SQL is not committed. Proceed to Task 2. @@ -345,7 +373,7 @@ without `&1`** (`TranscriptAttributes`, `GenomicSeqAttributes`, `DatasetPresente non-tuning base tables stay schema-qualified (`apidb.`, `sres.`). Compare the existing `GeneOrgAbbrev` and `GoSubsetLeaf` definitions in the same file. -- [ ] **Step 1: Insert the tuning table definition** +- [x] **Step 1: Insert the tuning table definition** Add this immediately after the closing `` of `TranscriptOrgAbbrev` (around line 1088). Note both `internalDependency` elements — omitting either causes a @@ -450,7 +478,7 @@ silently empty column rather than an error, per spec §3.3a. ``` -- [ ] **Step 2: Verify the file is still well-formed XML** +- [x] **Step 2: Verify the file is still well-formed XML** ```bash cd ~/workspaces/plasmodb/ApiCommonModel && python3 -c " @@ -461,7 +489,7 @@ print('XML OK')" Expected: `XML OK`. A malformed edit here breaks every tuning table, not just this one. -- [ ] **Step 3: Confirm the definition is registered exactly once** +- [x] **Step 3: Confirm the definition is registered exactly once** ```bash cd ~/workspaces/plasmodb/ApiCommonModel && \ @@ -470,7 +498,7 @@ cd ~/workspaces/plasmodb/ApiCommonModel && \ Expected: `1`. -- [ ] **Step 4: Commit** +- [x] **Step 4: Commit** ```bash cd ~/workspaces/plasmodb/ApiCommonModel @@ -492,7 +520,7 @@ built out of order these yield silently empty columns rather than errors." **Files:** none (build + verification) -- [ ] **Step 1: Build the table as a `jbrestel`-schema stub** +- [x] **Step 1: Build the table as a `jbrestel`-schema stub** `tuningManager` is not installed on this dev instance (`ls /var/www/jbrestel.plasmodb.org/gus_home/bin | grep -i tuning` returns only @@ -516,7 +544,7 @@ read it. Budget minutes: the aggregates cover 6.79M `VariationEffect` and 4.6M **Writes are confined to the `jbrestel` schema.** Never create or alter anything in `apidbtuning`, and never invent a `tuningManager` command line against a shared schema. -- [ ] **Step 1a: Point the model at the stub, in exactly six places** +- [x] **Step 1a: Point the model at the stub, in exactly six places** While stubbed, every query in Tasks 4-11 reads `jbrestel.VariationAttributes` instead of `ApidbTuning.VariationAttributes`. There are **six** occurrences across the two query @@ -540,7 +568,7 @@ the top of **both** query files so the stub cannot be forgotten: table. Flip back per Task 14 before merging. --> ``` -- [ ] **Step 1b: Verify the stub is complete and consistent** +- [x] **Step 1b: Verify the stub is complete and consistent** ```bash cd ~/workspaces/plasmodb/ApiCommonModel/Model/lib/wdk/model/records && \ @@ -561,7 +589,7 @@ plus 1 comment; `variationTableQueries.xml` has 2 (`TranscriptProducts`, A non-zero `ApidbTuning` count while stubbed means a query will fail at build time; a non-zero `jbrestel` count after Task 14 means the stub leaked into a merge. -- [ ] **Step 2: Verify row count matches the base table exactly** +- [x] **Step 2: Verify row count matches the base table exactly** ```bash psql -h localhost -p 5432 -d unidb_shu_a -c " @@ -572,7 +600,7 @@ select (select count(*) from jbrestel.variationattributes) as tuning, Expected: both `4390908`. A shortfall means the inner join to `GenomicSeqAttributes` dropped loci — investigate before continuing. -- [ ] **Step 3: Verify the aggregate columns are actually populated** +- [x] **Step 3: Verify the aggregate columns are actually populated** This is the assertion that catches a dependency-ordering failure, which produces empty columns rather than an error. @@ -591,7 +619,7 @@ Expected, approximately: `with_genes` ≈ 2,878,000 (not 0 — zero means the `TranscriptAttributes` dependency did not resolve), `with_snpeff` > `with_pc` (snpeff covers ~2.9M more loci), `with_allele` = 4,390,908, `projects` = 3. -- [ ] **Step 4: Verify the three spot-check loci survived the build** +- [x] **Step 4: Verify the three spot-check loci survived the build** ```bash psql -h localhost -p 5432 -d unidb_shu_a -x -c " @@ -604,7 +632,7 @@ order by location" Expected: identical to the Task 1 Step 2 table. -- [ ] **Step 5: No commit** (no files changed) +- [x] **Step 5: No commit** (no files changed) --- @@ -619,7 +647,7 @@ Everything after this task is additive and independently verifiable. - Create: `Model/lib/wdk/model/records/variationRecords.xml` - Modify: `Model/lib/wdk/apiCommonModel.xml:406-408` (add three imports near the snp ones) -- [ ] **Step 1: Create `variationAttributeQueries.xml`** +- [x] **Step 1: Create `variationAttributeQueries.xml`** ```xml @@ -696,7 +724,7 @@ Everything after this task is additive and independently verifiable. ``` -- [ ] **Step 2: Do NOT create `variationTableQueries.xml` yet** +- [x] **Step 2: Do NOT create `variationTableQueries.xml` yet** WDK's RELAX NG schema requires a `querySet` to contain at least one `sqlQuery`, `processQuery`, or `testRowCountSql`. A `querySet` holding only `defaultTestParamValues` @@ -710,7 +738,7 @@ element "querySet" incomplete; expected element "defaultTestParamValues", There is therefore no valid empty placeholder. `variationTableQueries.xml` is created in **Task 10**, together with its first query and its import line. -- [ ] **Step 3: Create `variationRecords.xml` with identity attributes only** +- [x] **Step 3: Create `variationRecords.xml` with identity attributes only** ```xml @@ -802,7 +830,7 @@ There is therefore no valid empty placeholder. `variationTableQueries.xml` is cr ``` -- [ ] **Step 4: Add the imports — mind the comment block** +- [x] **Step 4: Add the imports — mind the comment block** **The snp imports at lines 405-411 sit inside a `` block added in Task 3 Step 1a from the top of `variationAttributeQueries.xml` and `variationTableQueries.xml`. -- [ ] **Step 4: Verify no stub reference survives** +- [x] **Step 4: Verify no stub reference survives** ```bash cd ~/workspaces/plasmodb/ApiCommonModel && \ @@ -2177,7 +2205,7 @@ cd ~/workspaces/plasmodb/ApiCommonModel/Model/lib/wdk/model/records && \ Expected: `4` and `2`. -- [ ] **Step 5: Rebuild and re-verify** +- [x] **Step 5: Rebuild and re-verify** ```bash cd ~/workspaces/agentic-veupath-dev && bin/veup-build.sh plasmodb wb model @@ -2186,7 +2214,7 @@ cd ~/workspaces/agentic-veupath-dev && bin/veup-build.sh plasmodb wb model Then re-run the Task 13 Step 2 walk of all three variant-type records, confirming the pages are unchanged from the stubbed state. -- [ ] **Step 6: Commit** +- [x] **Step 6: Commit** ```bash cd ~/workspaces/plasmodb/ApiCommonModel @@ -2199,7 +2227,7 @@ ApidbTuning.VariationAttributes now that the tuning job has built it. Record pages verified unchanged across SNV, INDEL, and MIXED loci." ``` -- [ ] **Step 7: Drop the stub table** (optional, and only after Step 5 passes) +- [x] **Step 7: Drop the stub table** (optional, and only after Step 5 passes) ```bash psql -h localhost -p 5432 -d unidb_shu_a -c "DROP TABLE jbrestel.VariationAttributes" From 5fd66cae2dee589ade7877ffa9824e908873a78b Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Wed, 5 Aug 2026 10:04:13 -0400 Subject: [PATCH 24/98] Add the variation searches design spec (scaffolding + ID search) Covers the question-side scaffolding for the variation record -- a variationParams paramSet, a VariationsBy querySet, a VariationQuestions questionSet, and their apiCommonModel.xml imports -- plus the first search, VariationBySourceId, ported from the deprecated NgsSnpBySourceId. Settled decisions worth the reader's time: the nine-project includeProjects list is deliberate and must not be narrowed to the subset appDb's loaded three; ID matching is exact source_id only, with the alias query named as the home for any future tolerance; the question inherits the record's default summary columns rather than overriding them. The four remaining snp searches are out of scope and explained: they are HSSS processQuery searches over per-strain data, which is the seam the record spec deferred. Co-Authored-By: Claude Opus 5 --- .../2026-08-05-variation-searches-design.md | 291 ++++++++++++++++++ 1 file changed, 291 insertions(+) create mode 100644 docs/superpowers/specs/2026-08-05-variation-searches-design.md diff --git a/docs/superpowers/specs/2026-08-05-variation-searches-design.md b/docs/superpowers/specs/2026-08-05-variation-searches-design.md new file mode 100644 index 0000000000..c5c6f9b81d --- /dev/null +++ b/docs/superpowers/specs/2026-08-05-variation-searches-design.md @@ -0,0 +1,291 @@ +# Variation searches — design (scaffolding + `VariationBySourceId`) + +**Date:** 2026-08-05 +**Status:** approved +**Scope:** The question-side scaffolding for the `variation` record — a `paramSet`, a +`querySet`, a `questionSet`, and their model imports — plus the first and simplest +search, `VariationBySourceId`. The four remaining ported searches are **out of scope**; +see §8. +**Companion spec:** `2026-07-30-variation-record-design.md` (record class, attributes, +tuning table, record-page tables). Section references below prefixed `record §` point +there. +**Implementation target:** `ApiCommonModel` (branch `dnaseq-merge-experiments`) + +## 1. Purpose + +The `variation` record shipped with attributes and record-page tables but **no way to +search for one**. Today a variation is reachable only by URL or by a record-page link +from elsewhere. This adds the first search, and — more importantly — the three model +files and the import block that every later variation search will extend. + +The reference implementation is the deprecated `snp` record's `NgsSnpBySourceId` +(`Model/lib/wdk/model/questions/snpQuestions.xml`, +`questions/queries/snpQueries.xml`, `questions/params/snpParams.xml`). + +### 1.1 The reference is dead code + +`apiCommonModel.xml` has the entire snp block commented out (` + + + + + Input a comma delimited set of Variation IDs, or upload a file + + + + + + + + + + +``` + +- [ ] **Step 2: Create the querySet file** + +Write `Model/lib/wdk/model/questions/queries/variationQueries.xml`: + +```xml + + + + + + + + + + + + + + + + + + + + +``` + +- [ ] **Step 3: Create the questionSet file** + +Write `Model/lib/wdk/model/questions/variationQuestions.xml`: + +```xml + + + + + + + + + + + + Find variations by ID. + + + +
+ + Either enter the ID list manually, or upload a file that contains the list. + IDs can be delimited by a comma, a semi colon, or any white spaces. + ]]> + + + + + + + +``` + +- [ ] **Step 4: Verify all three files are well-formed XML** + +```bash +cd ~/workspaces/plasmodb/ApiCommonModel/Model/lib/wdk/model && \ + xmllint --noout questions/params/variationParams.xml \ + questions/queries/variationQueries.xml \ + questions/variationQuestions.xml && echo "XML OK" +``` + +Expected: `XML OK` and nothing else. Any `parser error` output means a typo — fix it +before continuing; the remote build takes minutes and will only tell you the same thing. + +- [ ] **Step 5: Verify the files reference only things that exist** + +```bash +cd ~/workspaces/plasmodb/ApiCommonModel/Model/lib/wdk/model && \ + grep -c 'name="VariationRecordClass"' records/variationRecords.xml && \ + grep -c 'name="VariationAlias"' records/variationAttributeQueries.xml +``` + +Expected: `1` and `1`. This confirms `recordClassRef="VariationRecordClasses.VariationRecordClass"` +resolves and that the record class this search targets is the one the record spec built. + +- [ ] **Step 6: Verify no personal identifiers leaked in** + +```bash +cd ~/workspaces/plasmodb/ApiCommonModel/Model/lib/wdk/model && \ + grep -n 'jbrestel' questions/params/variationParams.xml \ + questions/queries/variationQueries.xml \ + questions/variationQuestions.xml ; echo "exit: $?" +``` + +Expected: no matches, `exit: 1`. **Any match is a blocker for merge** — the same gate the +record plan used, because a hand-written stub schema name is easy to leave behind. + +- [ ] **Step 7: Commit** + +```bash +cd ~/workspaces/plasmodb/ApiCommonModel +git add Model/lib/wdk/model/questions/params/variationParams.xml \ + Model/lib/wdk/model/questions/queries/variationQueries.xml \ + Model/lib/wdk/model/questions/variationQuestions.xml +git commit -m "Add variation search scaffolding and the VariationBySourceId search + +Three new files -- a variationParams paramSet holding the ID datasetParam, +a VariationsBy querySet, and a VariationQuestions questionSet -- giving the +variation record its first search, ported from the deprecated snp record's +NgsSnpBySourceId. + +Not yet imported into apiCommonModel.xml, so this commit changes nothing +that builds. + +Co-Authored-By: Claude Opus 5 " +``` + +--- + +### Task 2: Import the files and confirm the model loads + +**Files:** +- Modify: `Model/lib/wdk/apiCommonModel.xml` (the "Variations" block, currently lines + 413–416) + +- [ ] **Step 1: Read the current import block to confirm the insertion point** + +```bash +cd ~/workspaces/plasmodb/ApiCommonModel/Model/lib/wdk && \ + grep -n 'variation' apiCommonModel.xml +``` + +Expected: exactly three matches, the record-side imports — +`model/records/variationAttributeQueries.xml`, `model/records/variationTableQueries.xml`, +`model/records/variationRecords.xml` — at lines 414, 415, 416. If the line numbers differ, +use the ones grep reports; insert after the `variationRecords.xml` line either way. + +- [ ] **Step 2: Add the three imports** + +In `Model/lib/wdk/apiCommonModel.xml`, replace: + +```xml + + + + +``` + +with: + +```xml + + + + + + + +``` + +Params before queries before questions — the order every other record's block uses +(compare the `spanParams`/`spanQueries`/`spanQuestions` trio just above). + +- [ ] **Step 3: Verify the edit is well-formed and complete** + +```bash +cd ~/workspaces/plasmodb/ApiCommonModel/Model/lib/wdk && \ + xmllint --noout apiCommonModel.xml && grep -c 'variation' apiCommonModel.xml +``` + +Expected: no xmllint output, then `6`. + +- [ ] **Step 4: Build the model on the remote** + +```bash +cd ~/workspaces/agentic-veupath-dev && bin/veup-build.sh plasmodb wb model +``` + +Expected: the build runs to completion and reloads the webapp. `wb model` (not +`ontology`) deliberately — this step proves the *model* references resolve, isolating an +XML mistake from an ontology mistake. The ontology row comes in Task 3. + +If it fails, the useful signal is in the model-load stack trace: an unresolved +`queryRef`/`paramRef`/`recordClassRef` names the exact broken reference. Fix and re-run +before going on. + +- [ ] **Step 5: Confirm the search is registered for this site** + +Open an already-authenticated page on `https://jbrestel.plasmodb.org` (any app page — +the session cookie is what makes this work; a raw `curl` 307-redirects to autologin), +then via Claude in Chrome `javascript_tool`: + +```javascript +fetch('/service/record-types/variation') + .then(r => r.json()) + .then(d => console.log(JSON.stringify(d.searches.map(s => s.fullName), null, 2))) +``` + +Expected: the array contains `VariationQuestions.VariationBySourceId`. + +This endpoint **is** project-filtered, which is why it is the source of truth for "does +this site have the search" — unlike the category tree checked in Task 3. + +- [ ] **Step 6: Commit** + +```bash +cd ~/workspaces/plasmodb/ApiCommonModel +git add Model/lib/wdk/apiCommonModel.xml +git commit -m "Import the variation question, query, and param files + +Wires the new variation search files into the model. wb model succeeds and +/service/record-types/variation now lists +VariationQuestions.VariationBySourceId. + +The search is not yet categorized, so it will not appear in the searches +menu until the individuals.txt row lands. + +Co-Authored-By: Claude Opus 5 " +``` + +--- + +### Task 3: Place the search in the category ontology + +Until this lands the search exists in the model but appears nowhere in the UI's search +menu, because the menu is built from the OWL. + +**Files:** +- Modify: `Model/lib/wdk/ontology/individuals.txt` (append after line 1157) + +- [ ] **Step 1: Confirm the file's column layout and the variation block's end** + +```bash +cd ~/workspaces/plasmodb/ApiCommonModel/Model/lib/wdk/ontology && \ + head -1 individuals.txt | tr '\t' '\n' | cat -n && \ + tail -1 individuals.txt && wc -l individuals.txt +``` + +Expected: 14 columns — +1 (blank, the subject id), 2 (blank, parent IRI), 3 (blank, parent label), +4 `recordClassName`, 5 `targetType`, 6 `name`, 7 `displayName`, 8 `shortDisplayName`, +9 `description`, 10 `geneOrTranscript`, 11 `displayOrder`, 12–14 `scope`. +The last line is `...VariationRecordClass.PredictedEffects`, and the count is `1157`. The +variation block runs 1104–1157 and ends the file, so the new row is appended. + +- [ ] **Step 2: Append the search row** + +This file is **tab-delimited** and the empty columns matter, so append it with a command +rather than by hand — a mistyped run of spaces silently shifts `menu` into the +`displayOrder` column: + +```bash +cd ~/workspaces/plasmodb/ApiCommonModel/Model/lib/wdk/ontology && \ +printf '%s\t%s\t%s\t%s\t%s\t%s\t\t\t\t\t\t%s\t%s\t\n' \ + 'VariationRecordClasses.VariationRecordClass.VariationQuestions.VariationBySourceId' \ + 'http://edamontology.org/topic_0199' \ + 'Genetic Variation' \ + 'VariationRecordClasses.VariationRecordClass' \ + 'search' \ + 'VariationQuestions.VariationBySourceId' \ + 'menu' \ + 'webservice' \ + >> individuals.txt +``` + +Parent node `topic_0199` / "Genetic Variation" and the `menu` + `webservice` scope pair +are what `PopsetQuestions.PopsetByPopsetId` uses, and `topic_0199` is already the parent +of the record's gene-linkage and effect-rollup attributes (record §8). + +- [ ] **Step 3: Verify the row has the right shape** + +```bash +cd ~/workspaces/plasmodb/ApiCommonModel/Model/lib/wdk/ontology && \ + tail -1 individuals.txt | awk -F'\t' '{print NF" fields"; print "5="$5; print "6="$6; print "12="$12; print "13="$13}' +``` + +Expected exactly: + +``` +14 fields +5=search +6=VariationQuestions.VariationBySourceId +12=menu +13=webservice +``` + +A field count other than 14, or `menu` landing anywhere but column 12, means the `printf` +was edited — redo Step 2 verbatim. + +- [ ] **Step 4: Rebuild the ontology** + +```bash +cd ~/workspaces/agentic-veupath-dev && bin/veup-build.sh plasmodb wb ontology +``` + +Expected: completes and reloads the webapp. **`wb ontology`, not `wb model`** — this is a +categorization change, and `wb model` would leave the OWL stale with no error anywhere. +`wb ontology` is a superset of `wb model`, so it is the only build needed here. + +- [ ] **Step 5: Confirm placement in the assembled tree** + +From an authenticated app page, via `javascript_tool`: + +```javascript +fetch('/service/ontologies/Categories') + .then(r => r.json()) + .then(d => { + const hits = []; + (function walk(node, parent) { + const p = node.properties || {}; + if ((p.name || []).includes('VariationQuestions.VariationBySourceId')) { + const pp = (parent && parent.properties) || {}; + hits.push({ + parentLabel: pp.label, + parentTerm: pp['EuPathDB alternative term'], + targetType: p.targetType + }); + } + (node.children || []).forEach(c => walk(c, node)); + })(d.tree, null); + console.log(JSON.stringify(hits, null, 2)); + }) +``` + +Expected: exactly one hit, with `targetType` `["search"]` and the parent's +`EuPathDB alternative term` reading `Genetic Variation`. + +Remember this endpoint is **not** project-filtered — use it for placement only. Presence +for this site was already established in Task 2 Step 5. + +- [ ] **Step 6: Commit** + +```bash +cd ~/workspaces/plasmodb/ApiCommonModel +git add Model/lib/wdk/ontology/individuals.txt +git commit -m "Categorize the VariationBySourceId search under Genetic Variation + +Places the search under edamontology topic_0199 with menu and webservice +scopes, matching the Popset ID search and the parent already used by the +variation record's gene-linkage and effect-rollup attributes. + +Verified via /service/ontologies/Categories after wb ontology. + +Co-Authored-By: Claude Opus 5 " +``` + +--- + +### Task 4: Verify the search actually works + +No code changes — this is the rung of the ladder that catches what the model and service +cannot: whether the SQL returns the right rows and the page renders. Do not skip it on the +grounds that the build passed. + +**Files:** none. + +- [ ] **Step 1: Mark the logs** + +```bash +cd ~/workspaces/agentic-veupath-dev && bin/veup-logs.sh plasmodb mark variation-search-qa +``` + +- [ ] **Step 2: Render the assembled SQL and the param list** + +```bash +cd ~/workspaces/agentic-veupath-dev && \ +ssh "$(python3 bin/resolve.py --profile profiles/plasmodb.yml --field host)" \ + 'bash -lc "source /var/www/jbrestel.plasmodb.org/etc/setenv && \ + wdkQuery -model PlasmoDB -query VariationsBy.VariationBySourceId -showParams"' +``` + +Expected: reports the single param `variation_id`. + +Then attempt the SQL render: + +```bash +cd ~/workspaces/agentic-veupath-dev && \ +ssh "$(python3 bin/resolve.py --profile profiles/plasmodb.yml --field host)" \ + 'bash -lc "source /var/www/jbrestel.plasmodb.org/etc/setenv && \ + wdkQuery -model PlasmoDB -query VariationsBy.VariationBySourceId -showQuery"' +``` + +**This one is allowed to fail, and a failure is not a defect.** `$$variation_id$$` is a +`datasetParam` macro that expands to a subquery against the user-dataset tables, which +needs a dataset ID that only exists once a user has actually submitted the form — there +is nothing for the CLI to substitute. If it errors on the missing param value, record +that in the task notes and move on; Steps 3–5 are the real test of the SQL. If it *does* +render, read the SQL and confirm it selects `source_id`/`project_id` from +`ApidbTuning.VariationAttributes`. + +(Both commands need the concrete docroot. `/var/www/jbrestel.plasmodb.org` above assumes +`user: jbrestel`; get it for the current identity with +`python3 bin/resolve.py --profile profiles/plasmodb.yml --field docroot`.) + +- [ ] **Step 3: Find the search in the site's search menu and open it** + +In Claude in Chrome, load `https://jbrestel.plasmodb.org`, open the searches menu, and +find **"Variation ID(s)"** under the Variation record type. This is the user-visible +payoff of Task 3 and the one thing none of the service checks prove — so navigate here +rather than deep-linking. + +(The direct URL is roughly +`//app/search/variation/VariationBySourceId`, but the context path segment +is site-specific and is not recorded in this instance's `docs/app.md`. Read the real URL +off the address bar once the menu gets you there, and note it in `docs/app.md` under "Key +pages" so the next session can deep-link.) + +Expected: the form renders with an ID input box prefilled from the `suggest` default +(`Variant_Pf3D7_01_v3_100057`), labelled "Variation ID input set", with manual-entry and +file-upload options. + +Submit it. Expected: lands **directly on the record page** for +`Variant_Pf3D7_01_v3_100057` with no intervening result page — that is +`noSummaryOnSingleRecord="true"` working. + +- [ ] **Step 4: Multi-ID search shows the record's default columns, and tolerates a bad ID** + +Go back to the search and submit these four IDs, comma-delimited — three real, one +deliberately bogus: + +``` +Variant_Pf3D7_01_v3_100057, Variant_Pf3D7_01_v3_29514, Variant_Pf3D7_01_v3_12, Variant_Nonexistent_9999 +``` + +Expected: a result page with **3 rows**, not 4. An unmatched ID is silently absent, which +is the `datasetParam` norm; no error is shown. + +Confirm the columns are the record's defaults (record §6.9), i.e. that no +`attributesList` override crept in: Location, Gene ID(s), Variant Type, the collapsed +allele and collapsed minor allele frequency, both `most_severe_impact_*` columns, and +strain count. + +`Variant_Pf3D7_01_v3_12` is the `MIXED` locus documented in record §2, so its collapsed +allele should show both classes joined with `; ` (e.g. `A>C; A>AC`) — a free check that +the search feeds the record's attribute layer correctly rather than a stripped-down one. + +- [ ] **Step 5: Confirm nothing broke** + +```bash +cd ~/workspaces/agentic-veupath-dev && bin/veup-logs.sh plasmodb since variation-search-qa --quiet +``` + +Expected: the error logs (`svc-error`, `wdk`, `error`, `catalina`) report `silent:`. Page +views and service access lines for the searches are expected and fine. + +- [ ] **Step 6: Record the results in the plan** + +Tick the boxes above and note the Step 2 `-showQuery` outcome (rendered vs. failed on the +`datasetParam` macro) inline, so the next person does not re-litigate it. + +--- + +### Task 5: Update the spec's status and close out + +**Files:** +- Modify: `docs/superpowers/specs/2026-08-05-variation-searches-design.md` (the `Status:` + line) +- Modify: `docs/superpowers/plans/2026-08-05-variation-searches.md` (this file, checkboxes) + +- [ ] **Step 1: Confirm the whole change set is exactly what was intended** + +```bash +cd ~/workspaces/plasmodb/ApiCommonModel && \ + git diff --stat HEAD~3 && grep -rn 'jbrestel' Model/lib/wdk/ ; echo "grep exit: $?" +``` + +Expected: five files touched — the three new question-side files, `apiCommonModel.xml`, +and `ontology/individuals.txt` — and **no `jbrestel` matches** (`grep exit: 1`). A match +in `Model/lib/wdk/` is a merge blocker. + +- [ ] **Step 2: Mark the spec implemented** + +In `docs/superpowers/specs/2026-08-05-variation-searches-design.md`, change: + +```markdown +**Status:** approved +``` + +to: + +```markdown +**Status:** implemented 2026-08-05 (scaffolding + `VariationBySourceId`; the four searches in §8.1 remain unstarted) +``` + +- [ ] **Step 3: Commit** + +```bash +cd ~/workspaces/plasmodb/ApiCommonModel +git add docs/superpowers/specs/2026-08-05-variation-searches-design.md \ + docs/superpowers/plans/2026-08-05-variation-searches.md +git commit -m "Mark the variation searches spec implemented + +Scaffolding and VariationBySourceId are built and verified on the plasmodb +dev instance. The four HSSS-backed searches remain out of scope pending the +per-strain seam. + +Co-Authored-By: Claude Opus 5 " +``` + +--- + +## Deliberately not in this plan + +- **The four remaining snp searches** — `NgsSnpsByIsolateGroup`, `NgsSnpsByLocation`, + `NgsSnpsByGeneIds`, `NgsSnpsByTwoIsolateGroups`(`Wiz`). Spec §8.1: they are HSSS + `processQuery` searches over per-strain data, which is the seam record §9 deferred. + Their spec's first question is what replaces HSSS, not what the XML looks like. +- **ID tolerance for coordinate-style input** — spec §5.1. If it is ever wanted, it goes + in `VariationAttributes.VariationAlias`, not in a search's SQL. +- **`` defaults for AmoebaDB, CryptoDB, MicrosporidiaDB, PiroplasmaDB, ToxoDB** — + spec §4.1. They need loaded data first; do not invent IDs. +- **Narrowing `includeProjects` to the three projects with loaded data.** Spec §2: all + nine are expected to have variation data in the full database. The empty ones are an + artifact of the subset dev appDb. +- **Verifying the empty-project case** (form renders, 0 rows, no error). Spec §7.3 step 7 + — it cannot be checked from the plasmodb instance, which has data. Run it when one of + those five sites is next stood up. +- **A sanity-test file.** `apiCommonModel-sanity.xml` imports a directory that does not + exist; the whole sanity model is dead. Do not extend it. +- **Uncommenting the dead snp block** to compare behavior. It references + `apidbtuning.SnpAttributes`, which does not exist in this build; the model would fail to + load. Spec §1.1. +- **Deleting the dead snp model XML.** Record §10: its own blast radius + (`recordParams.xml`, `spanQuestions.xml`, `SnpsBySpanLogic`). From 6130ef92f10e739f83fd1b5ca1ba67316a74961f Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Wed, 5 Aug 2026 10:28:51 -0400 Subject: [PATCH 26/98] Add variation search scaffolding and the VariationBySourceId search Three new files -- a variationParams paramSet holding the ID datasetParam, a VariationsBy querySet, and a VariationQuestions questionSet -- giving the variation record its first search, ported from the deprecated snp record's NgsSnpBySourceId. Not yet imported into apiCommonModel.xml, so this commit changes nothing that builds. Co-Authored-By: Claude Opus 5 --- .../questions/params/variationParams.xml | 27 ++++++++++++++ .../questions/queries/variationQueries.xml | 28 +++++++++++++++ .../model/questions/variationQuestions.xml | 36 +++++++++++++++++++ 3 files changed, 91 insertions(+) create mode 100644 Model/lib/wdk/model/questions/params/variationParams.xml create mode 100644 Model/lib/wdk/model/questions/queries/variationQueries.xml create mode 100644 Model/lib/wdk/model/questions/variationQuestions.xml diff --git a/Model/lib/wdk/model/questions/params/variationParams.xml b/Model/lib/wdk/model/questions/params/variationParams.xml new file mode 100644 index 0000000000..0c6bd51129 --- /dev/null +++ b/Model/lib/wdk/model/questions/params/variationParams.xml @@ -0,0 +1,27 @@ + + + + + + + + + + Input a comma delimited set of Variation IDs, or upload a file + + + + + + + + + + diff --git a/Model/lib/wdk/model/questions/queries/variationQueries.xml b/Model/lib/wdk/model/questions/queries/variationQueries.xml new file mode 100644 index 0000000000..e01f8db653 --- /dev/null +++ b/Model/lib/wdk/model/questions/queries/variationQueries.xml @@ -0,0 +1,28 @@ + + + + + + + + + + + + + + + + + + + + diff --git a/Model/lib/wdk/model/questions/variationQuestions.xml b/Model/lib/wdk/model/questions/variationQuestions.xml new file mode 100644 index 0000000000..f814240746 --- /dev/null +++ b/Model/lib/wdk/model/questions/variationQuestions.xml @@ -0,0 +1,36 @@ + + + + + + + + + + + + Find variations by ID. + + + +
+ + Either enter the ID list manually, or upload a file that contains the list. + IDs can be delimited by a comma, a semi colon, or any white spaces. + ]]> + + + + + + + From ff563a57065047c3d45f3bc8d9efc7fbfdc766c6 Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Wed, 5 Aug 2026 10:39:22 -0400 Subject: [PATCH 27/98] Disambiguate the spec reference in variationParams.xml "See spec section 5.1" did not say which of the two specs in docs/superpowers/specs/ it meant. The claim is accurate against the searches spec, but the record spec's section 5 has no subsections, so a maintainer who opens the record spec -- the likelier bookmark when editing this record -- finds nothing and concludes the comment is stale. Names the spec, matching the disambiguation style already used in variationQueries.xml. Comment-only; no element, attribute, value or SQL change. Co-Authored-By: Claude Opus 5 --- Model/lib/wdk/model/questions/params/variationParams.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Model/lib/wdk/model/questions/params/variationParams.xml b/Model/lib/wdk/model/questions/params/variationParams.xml index 0c6bd51129..32d6fa2999 100644 --- a/Model/lib/wdk/model/questions/params/variationParams.xml +++ b/Model/lib/wdk/model/questions/params/variationParams.xml @@ -9,7 +9,7 @@ + search's SQL, so every entry point benefits. See the searches spec section 5.1. --> From cf87f9d993562a7b5a2fe454343aa95ad8b37821 Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Wed, 5 Aug 2026 10:50:13 -0400 Subject: [PATCH 28/98] Import the variation question, query, and param files Wires the new variation search files into the model. wb model succeeds and the assembled PlasmoDB model now contains VariationQuestions.VariationBySourceId. The search is not yet categorized, so it will not appear in the searches menu until the individuals.txt row lands. Co-Authored-By: Claude Opus 5 --- Model/lib/wdk/apiCommonModel.xml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Model/lib/wdk/apiCommonModel.xml b/Model/lib/wdk/apiCommonModel.xml index f3a043c6d0..b63c1d166e 100644 --- a/Model/lib/wdk/apiCommonModel.xml +++ b/Model/lib/wdk/apiCommonModel.xml @@ -414,6 +414,9 @@ + + + + + + + + + + + + + + + +``` + +Note the column element is `parentTerm` (camel case) — that is what `withNgsSNPsTree` declares, and WDK matches it case-insensitively to the SQL's `parentterm`. + +- [ ] **Step 4: Verify the XML still parses** + +```bash +python3 -c "import xml.etree.ElementTree as T; T.parse('$HOME/workspaces/plasmodb/ApiCommonModel/Model/lib/wdk/model/questions/params/organismParams.xml'); print('parses')" +``` + +Expected: `parses`. (This catches an unbalanced tag in seconds instead of at the end of a five-minute remote build.) + +- [ ] **Step 5: Commit** + +```bash +cd ~/workspaces/plasmodb/ApiCommonModel +git add Model/lib/wdk/model/questions/params/organismParams.xml +git commit -m "Add organismVQ.withVariationsTree for the variation searches + +The snp original (withNgsSNPsTree) reads apidbtuning.snpstrains, which does +not exist in this build, so this is a rewrite. Filters on apidb.datasource's +declared type='isolates'/subtype='Dna_Seq' -- the 10 dnaseq experiment +datasets -- rather than a name string convention. + +internal is the taxon name, not an abbreviation: the HSSS plugin resolves it +through sres.TaxonName to get name_for_filenames for the webservices path." +``` + +--- + +### Task 3: `eda_sample_table_suffix` — the hidden dependent param + +The one non-obvious piece of the design. The organism param's internal value **must** be the taxon name (the plugin needs it); the EDA queries need the study+entity abbreviation to build table names. Two identities, one dropdown — resolved with a hidden param that carries the second identity. Design §2 and §4.2. + +**Files:** +- Modify: `Model/lib/wdk/model/questions/params/variationParams.xml` (new `querySet VariationVQ` + one param) + +- [ ] **Step 1: Run the vocabulary SQL — it must return exactly one row** + +```bash +psql -h localhost -p 5432 -d unidb_shu_a -c " +SELECT DISTINCT + s.internal_abbrev || '_' || lower(e.internal_abbrev) AS internal, + s.internal_abbrev || '_' || lower(e.internal_abbrev) AS term +FROM apidb.datasource ds +JOIN apidb.organism o ON o.taxon_id = ds.taxon_id +JOIN sres.taxonname tn ON tn.taxon_id = o.taxon_id AND tn.name_class = 'scientific name' +JOIN sres.externaldatabase ed ON ed.name = ds.name +JOIN sres.externaldatabaserelease edr ON edr.external_database_id = ed.external_database_id +JOIN eda.studyexternaldatabaserelease sedr ON sedr.external_database_release_id = edr.external_database_release_id +JOIN eda.study s ON s.study_id = sedr.study_id +JOIN eda.entitytypegraph e ON e.study_id = s.study_id +WHERE ds.type = 'isolates' AND ds.subtype = 'Dna_Seq' + AND tn.name = 'Plasmodium falciparum 3D7' + AND s.internal_abbrev IS NOT NULL" +``` + +Expected: exactly **one row**, `s3be28bbe14_sample`. More than one row means the tables interpolated downstream would be ambiguous — stop; the joins need narrowing before anything else is written. + +- [ ] **Step 2: Confirm the tables that name implies actually exist** + +```bash +psql -h localhost -p 5432 -d unidb_shu_a -tAc " +SELECT to_regclass('eda.attributevalue_s3be28bbe14_sample'), + to_regclass('eda.attributegraph_s3be28bbe14_sample')" +``` + +Expected: both non-null. This is the check that a *looked-up* abbreviation buys you and a recomputed hash does not — see design §3. + +- [ ] **Step 3: Add the `VariationVQ` querySet with this one query** + +In `variationParams.xml`, after the closing `` and before ``: + +```xml + + + + + + + + + + + + + + +``` + +`noTranslation="true"` on the organism `paramRef` makes WDK pass the param's **term** rather than its internal — for a selected leaf these are the same taxon name, and it is what `SnpVQ` did. The SQL supplies the quotes, so the param must not also be quoted. + +- [ ] **Step 4: Add the param itself** + +Inside the existing `variationParams` paramSet, after the `variation_id` `datasetParam`: + +```xml + + + + + + Derived from the selected organism. Not user-visible. + +``` + +- [ ] **Step 5: Verify the XML parses** + +```bash +python3 -c "import xml.etree.ElementTree as T; T.parse('$HOME/workspaces/plasmodb/ApiCommonModel/Model/lib/wdk/model/questions/params/variationParams.xml'); print('parses')" +``` + +Expected: `parses`. + +- [ ] **Step 6: Commit** + +```bash +cd ~/workspaces/plasmodb/ApiCommonModel +git add Model/lib/wdk/model/questions/params/variationParams.xml +git commit -m "Add hidden eda_sample_table_suffix param and VariationVQ querySet + +The organism param's internal value must be the taxon name -- the HSSS plugin +resolves it through sres.TaxonName to build the webservices path. The EDA +filter queries need the study+entity abbreviation to name per-study tables. +Two identities for one dropdown, so the second travels in a hidden dependent +param and gets interpolated into the table name. + +Looked up rather than recomputed from the SHA-1 convention: a lookup fails +visibly (empty dropdown), a stale hash fails invisibly (missing relation, +much later, from inside a filterParam)." +``` + +--- + +### Task 4: `variation_sample_meta` — the EDA-driven samples filter + +**Files:** +- Modify: `Model/lib/wdk/model/questions/params/variationParams.xml` (two queries into `VariationVQ`, one `filterParam`) + +> **The param name `variation_sample_meta` is a contract, not a style choice.** +> `FindPolymorphismsPlugin.getStrainFilterParamName()` returns exactly this string and +> `FindPolymorphismsAbstractPlugin` lists it among the plugin's **required** params. Any +> other spelling is rejected at run time as a missing required parameter. Do not "improve" it. + +- [ ] **Step 1: Run the metadata SQL** + +```bash +psql -h localhost -p 5432 -d unidb_shu_a -c " +SELECT count(*) AS rows, + count(DISTINCT av.sample_stable_id) AS samples, + count(DISTINCT av.attribute_stable_id) AS attributes, + count(av.string_value) AS strings, + count(av.number_value) AS numbers, + count(av.date_value) AS dates +FROM eda.attributevalue_s3be28bbe14_sample av" +``` + +Expected: `3771 | 216 | 20 | 2013 | 1758 | 0`. Zero dates is fine — no date-typed filters will appear, which is this dataset, not a defect. + +- [ ] **Step 2: Run the ontology SQL and check WDK's two throw conditions** + +WDK's `OntologyItemNewFetcher.validateOntologyItems` throws if (a) any node names a parent that is not itself a node, or (b) any node with a NULL type has no children. Check both before writing XML: + +```bash +psql -h localhost -p 5432 -d unidb_shu_a -c " +WITH t AS ( + SELECT ag.stable_id AS ontology_term_name, + CASE WHEN ag.parent_stable_id IN + (SELECT stable_id FROM eda.attributegraph_s3be28bbe14_sample) + THEN ag.parent_stable_id + END AS parent_ontology_term_name, + CASE ag.data_type + WHEN 'string' THEN 'string' + WHEN 'number' THEN 'number' + WHEN 'integer' THEN 'number' + END AS type + FROM eda.attributegraph_s3be28bbe14_sample ag +) +SELECT (SELECT count(*) FROM t) AS nodes, + (SELECT count(*) FROM t WHERE parent_ontology_term_name IS NULL) AS roots, + (SELECT count(*) FROM t WHERE parent_ontology_term_name IS NOT NULL + AND parent_ontology_term_name NOT IN (SELECT ontology_term_name FROM t)) AS dangling, + (SELECT count(*) FROM t WHERE type IS NULL AND ontology_term_name NOT IN + (SELECT parent_ontology_term_name FROM t WHERE parent_ontology_term_name IS NOT NULL)) AS childless_branches, + (SELECT count(*) FROM t WHERE type IS NOT NULL) AS typed_leaves" +``` + +Expected: `27 | 7 | 0 | 0 | 20`. + +**`dangling` and `childless_branches` must both be 0.** They are what the `CASE` on the parent is for: EDA's attribute graph has **no row for the entity itself**, so seven category nodes declare `parent_stable_id = 'sample'` and nothing has `stable_id = 'sample'`. Mapping unresolvable parents to NULL makes WDK adopt them under its own synthetic master root. + +- [ ] **Step 3: Prove the naive version would in fact fail** + +Worth ten seconds, because the `CASE` looks like noise until you see this: + +```bash +psql -h localhost -p 5432 -d unidb_shu_a -tAc " +SELECT count(*) FROM eda.attributegraph_s3be28bbe14_sample ag +WHERE ag.parent_stable_id IS NOT NULL + AND ag.parent_stable_id NOT IN (SELECT stable_id FROM eda.attributegraph_s3be28bbe14_sample)" +``` + +Expected: `7`. Those are seven guaranteed `WdkModelException`s if you write `ag.parent_stable_id` unguarded. + +- [ ] **Step 4: Confirm the filter's internal values are usable by HSSS** + +The filter hands sample stable IDs to the plugin, which writes them to a strains file with `strains_are_names = 1`. They must be names HSSS knows: + +```bash +psql -h localhost -p 5432 -d unidb_shu_a -tAc " +SELECT count(DISTINCT sample_stable_id) FROM eda.attributevalue_s3be28bbe14_sample" \ +&& ssh cedar "cut -f2 /var/www/Common/apiSiteFilesMirror/webServices/PlasmoDB/build-71/Pfalciparum3D7/dnaseq/readFreq20/strainIdToName.dat | sort -u | wc -l" +``` + +Expected: `216` and `538`. EDA covers a subset of the strains HSSS knows, which is the safe direction: the filter cannot offer a strain HSSS has never heard of. (The strict-subset relation was verified when the design was written; the counts here are the cheap re-check.) + +- [ ] **Step 5: Add both queries to `VariationVQ`** + +After `EdaSampleTableSuffix`, inside the `VariationVQ` querySet: + +```xml + + + + + + + + + + + + + + + + + + + + + + + + + + + + +``` + +- [ ] **Step 6: Add the `filterParam`** + +Inside `variationParams`, after `eda_sample_table_suffix`: + +```xml + + + + + + + Select a set of samples whose genomic sequences will be compared. Use the + sample characteristics to narrow the group, or accept all samples for the + organism you chose. + + +``` + +- [ ] **Step 7: Verify the XML parses** + +```bash +python3 -c "import xml.etree.ElementTree as T; T.parse('$HOME/workspaces/plasmodb/ApiCommonModel/Model/lib/wdk/model/questions/params/variationParams.xml'); print('parses')" +``` + +Expected: `parses`. + +- [ ] **Step 8: Commit** + +```bash +cd ~/workspaces/plasmodb/ApiCommonModel +git add Model/lib/wdk/model/questions/params/variationParams.xml +git commit -m "Add EDA-driven variation_sample_meta filterParam and its two queries + +The samples filter moves from apidbTuning.Metadata to EDA's per-study tables, +named by interpolating the hidden suffix param. + +The CASE on parent_stable_id is load-bearing, not defensive: EDA's attribute +graph has no row for the entity itself, so seven category nodes point at a +parent 'sample' that does not exist as a node, and WDK's validateOntologyItems +throws on precisely that. Mapping unresolvable parents to NULL hands them to +WDK's synthetic master root. Verified 27 nodes -> 7 roots, 0 dangling, 0 +childless branches, 20 typed leaves. + +The param name is the plugin's required-parameter contract; do not rename it." +``` + +--- + +### Task 5: The four HSSS path and threshold params + +Copied out of `snpParams.xml`, **not** referenced there. Design §4.4: `snpParams.xml` is imported inside the commented-out snp block, so the `snpParams` paramSet is absent from the assembled model and every `paramRef` to it would fail model load. Uncommenting is not the fix either — that file references `SnpRecordClasses.SnpRecordClass`, also commented out. + +**Files:** +- Modify: `Model/lib/wdk/model/questions/params/variationParams.xml` + +- [ ] **Step 1: Confirm for yourself that `snpParams` is not in the assembled model** + +Do not take the plan's word for it, and do not use `grep` on `apiCommonModel.xml` — grep cannot tell a live import from a commented-out one: + +```bash +ssh cedar 'bash -lc "source /var/www/jbrestel.plasmodb.org/etc/setenv && wdkXml -model PlasmoDB"' \ + | grep -c "paramSet name=\"snpParams\"" +``` + +Expected: `0`. That zero is the whole justification for copying rather than referencing. + +- [ ] **Step 2: Confirm the four read-frequency directories exist, since the internals must match them** + +```bash +ssh cedar 'ls /var/www/Common/apiSiteFilesMirror/webServices/PlasmoDB/build-71/Pfalciparum3D7/dnaseq/' +``` + +Expected: `readFreq20 readFreq40 readFreq60 readFreq80` (plus `bigwig` and `vcf`, which are out of scope). The four `ReadFrequencyPercent` internals below must be exactly `20`/`40`/`60`/`80`. + +- [ ] **Step 3: Add the four params** + +Inside `variationParams`, after `variation_sample_meta`: + +```xml + + + + + + + + + + dflt + @WEBSERVICEMIRROR@/PROJECT_GOES_HERE/build-%%buildNumber%% + + + + + + + + This parameter applies to the sequencing reads of individual samples and + defines a stringency for data supporting a variant call between a sample and + the reference genome (Organism). Each nucleotide position of each sample is + compared to the reference genome and a call is made if the portion of the + sample's aligned reads that support the variant is above the Read Frequency + Threshold (RFT). Find high quality haploid variants with 80% RFT or + heterozygous diploid/aneuploid variants with 40%. See the Description below + for more. + + + + 80% + 80 + + + 60% + 60 + + + 40% + 40 + + + 20% + 20 + + + + + + + This parameter applies to your group of samples. A variant can occur in any + number of samples in your group and the least frequent call across all + samples is the Minor Allele Frequency. A variant will be returned by the + search if the frequency of the minor allele is equal to or greater than your + Minor Allele Frequency. See the Description below the Get Answer button for + more. + + + \d\d? + + + + + This parameter applies to the selected set of aligned sample sequences. At + any given nucleotide position, some samples in your group may not have data + supporting a base call because the Read Frequency Threshold was not met or + fewer than our minimum of 5 reads aligned. 'Percent samples with a base call' + defines the fraction of the selected samples that must have a base call + before a variant is returned for that nucleotide position, based on the + remaining samples that do have data. See the Description below for more + information. + + + \d\d?|100 + +``` + +The param **names** stay as the originals (including `MinPercentIsolateCalls`) — the plugin reads them by name. Only prompts and help text move from "isolates" to "samples". + +- [ ] **Step 4: Verify the XML parses and the regexes survived escaping** + +```bash +python3 - <<'PY' +import xml.etree.ElementTree as T, os +p = os.path.expanduser('~/workspaces/plasmodb/ApiCommonModel/Model/lib/wdk/model/questions/params/variationParams.xml') +r = T.parse(p).getroot() +for sp in r.iter('stringParam'): + rx = sp.find('regex') + print(sp.get('name'), '->', repr(rx.text if rx is not None else None)) +PY +``` + +Expected exactly: +``` +MinPercentMinorAlleles -> '\\d\\d?' +MinPercentIsolateCalls -> '\\d\\d?|100' +``` +(Python shows a literal backslash as `\\`. If you see `\\\\d`, the backslashes got doubled — fix it; the regex would reject every input.) + +- [ ] **Step 5: Build, and prove the model loads with all five new params** + +This is the first remote build. It takes a few minutes. + +```bash +cd ~/workspaces/agentic-veupath-dev && bin/veup-build.sh plasmodb wb model +``` + +Expected: completes without a `WdkModelException`. If it fails, the message names the unresolved reference — that is the whole value of this step. + +Then prove the params are in the *assembled* model, not merely in a file: + +```bash +ssh cedar 'bash -lc "source /var/www/jbrestel.plasmodb.org/etc/setenv && wdkXml -model PlasmoDB"' \ + | grep -E "name=\"(eda_sample_table_suffix|variation_sample_meta|WebServicesPath|ReadFrequencyPercent|MinPercentMinorAlleles|MinPercentIsolateCalls)\"" +``` + +Expected: one matching line per name, six in all. `wb model` is correct here — nothing has touched categorization yet; Task 8 is what forces `wb ontology`. + +- [ ] **Step 6: Commit** + +```bash +cd ~/workspaces/plasmodb/ApiCommonModel +git add Model/lib/wdk/model/questions/params/variationParams.xml +git commit -m "Copy the four HSSS path and threshold params into variationParams + +Not referenced from snpParams: that file is imported inside the commented-out +snp block, so the snpParams paramSet is absent from the assembled model +(wdkXml finds it nowhere) and every paramRef to it would fail model load. +Uncommenting the one import fails too -- the file references +SnpRecordClasses.SnpRecordClass, also commented out. The originals are +unimported dead code, so this is migration out of a dead file. + +Two deliberate departures: one ReadFrequencyPercent rather than the original's +two (they differed only in help text phrased for the two-group search), and +help text saying 'samples' to match the EDA vocabulary the filter is built +from. Param names are unchanged -- the plugin reads them by name." +``` + +--- + +### Task 6: The `processQuery` + +**Files:** +- Modify: `Model/lib/wdk/model/questions/queries/variationQueries.xml` + +- [ ] **Step 1: Confirm the plugin class and its required params before wiring to them** + +```bash +cd ~/workspaces/plasmodb/ApiCommonWebService +grep -rn "getStrainFilterParamName\|PARAM_ORGANISM\|REQUIRED" \ + WSFPlugin/lib/java/org/apidb/apicomplexa/wsfplugin/highspeedsnpsearch/FindPolymorphismsPlugin.java \ + WSFPlugin/lib/java/org/apidb/apicomplexa/wsfplugin/highspeedsnpsearch/FindPolymorphismsAbstractPlugin.java \ + | head -20 +``` + +Expected: `FindPolymorphismsPlugin` returns `"variation_sample_meta"` from `getStrainFilterParamName()`, and the abstract plugin lists that name among its required params. If the returned string is anything else, **stop** — Task 4's param name must match it, and the plan is stale. + +- [ ] **Step 2: Add the query** + +In `variationQueries.xml`, inside the `VariationsBy` querySet, after the `VariationBySourceId` `sqlQuery`: + +```xml + + + + + + + + The Organism defines the species identity of the samples and the genome + against which each sample's variants were called. After choosing an Organism, + the set of samples available for forming groups is limited to samples aligned + to your chosen Organism's genome. + The organism you choose determines the genome to which the + variants have been mapped. It also restricts the set of samples you may + choose, since variants are identified by aligning that sample's reads to this + genome. + + + + + + + + + + + + + + + +``` + +- [ ] **Step 3: Verify the XML parses** + +```bash +python3 -c "import xml.etree.ElementTree as T; T.parse('$HOME/workspaces/plasmodb/ApiCommonModel/Model/lib/wdk/model/questions/queries/variationQueries.xml'); print('parses')" +``` + +Expected: `parses`. + +- [ ] **Step 4: Commit** (no build yet — Task 7's question is what makes this query reachable, and one build covers both) + +```bash +cd ~/workspaces/plasmodb/ApiCommonModel +git add Model/lib/wdk/model/questions/queries/variationQueries.xml +git commit -m "Add VariationsByIsolateGroup process query + +Binds FindPolymorphismsPlugin to the new variation params. The wsColumn set is +dictated by the plugin, which throws unless its results file has exactly four +tab-separated columns; project_id comes from the plugin rather than the file. +organism quote=false is required for the dependent-param query." +``` + +--- + +### Task 7: The question + +**Files:** +- Modify: `Model/lib/wdk/model/questions/variationQuestions.xml` + +- [ ] **Step 1: Confirm the summary attributes exist on the record** + +An `attributesList` naming an attribute the record does not have fails model load: + +```bash +cd ~/workspaces/plasmodb/ApiCommonModel +grep -cE 'name="(variation_location|gene_ids|variant_type)"' Model/lib/wdk/model/records/variationRecords.xml +``` + +Expected: `3`. The other three summary columns (`PercentMinorAlleles`, `PercentIsolateCalls`, `Phenotype`) are the query's dynamic `wsColumn`s from Task 6 and will not appear in the record file — that is correct. + +- [ ] **Step 2: Add the question** + +In `variationQuestions.xml`, inside the `VariationQuestions` questionSet, after `VariationBySourceId`: + +```xml + + + + + + + + + + + + + +
+ + Each sample's sequencing reads are aligned to the reference genome + (Organism) and variants are recorded for each sample based on the Read + Frequency Threshold. Then, scanning variant locations across the group of + samples, variants are returned by the search if the Minor Allele Frequency + and the Percent samples with a base call are met. + +

Organism: The Organism parameter defines the species of the + samples and the genome in which the variants are determined. Choosing an + Organism focuses the Samples parameter to the samples of that organism, + changing the subset available when forming your group.

+ +

Samples: Sample sequences are accompanied by characteristics of + the sample -- where it was collected, the host, alignment statistics. By + default the group includes all samples from the Organism you chose; you may + narrow the group using those characteristics. At least two samples are + required, since polymorphism within a group of one is undefined.

+ +

Read frequency threshold: An allele is called for a sample at a + location if that fraction of the sample's aligned reads support it. For + example, a sample with 10 reads at a location -- 6 A and 4 C -- is called A + at a threshold of 60% or less, and not called at 80%. This matters most for + diploid or aneuploid organisms, where heterozygous positions are expected + near 50%.

+ +

Minor allele frequency: Among the qualifying calls at a location, + the minor allele frequency is the percent that are not the major allele. A + location is returned if that is at or above the value you specify. Use 0 to + find every variant location within the group.

+ +

Percent samples with a base call: A location is only considered if + this fraction of your selected samples have a qualifying call there. With 20 + samples and a threshold of 75%, a location with fewer than 15 called samples + is ignored.

+ ]]> + + + +``` + +- [ ] **Step 3: Verify the XML parses** + +```bash +python3 -c "import xml.etree.ElementTree as T; T.parse('$HOME/workspaces/plasmodb/ApiCommonModel/Model/lib/wdk/model/questions/variationQuestions.xml'); print('parses')" +``` + +Expected: `parses`. + +- [ ] **Step 4: Build and prove the question is in the assembled model** + +```bash +cd ~/workspaces/agentic-veupath-dev && bin/veup-build.sh plasmodb wb model +``` + +Expected: completes without a `WdkModelException`. + +```bash +ssh cedar 'bash -lc "source /var/www/jbrestel.plasmodb.org/etc/setenv && wdkXml -model PlasmoDB"' \ + | grep -c "VariationsByIsolateGroup" +``` + +Expected: a non-zero count. Zero means the question is in the file but not in the model for PlasmoDB — check `includeProjects` on the enclosing questionSet. + +- [ ] **Step 5: Commit** + +```bash +cd ~/workspaces/plasmodb/ApiCommonModel +git add Model/lib/wdk/model/questions/variationQuestions.xml +git commit -m "Add the VariationsByIsolateGroup question + +Overrides attributesList, unlike VariationBySourceId: the plugin's dynamic +minor-allele-frequency and percent-called columns are the point of the search +and are not in the record's default summary. + +displayName says Samples rather than the snp original's Isolates, matching the +EDA vocabulary the filter is built from. noSummaryOnSingleRecord deliberately +unset -- a one-hit analytical result wants its context." +``` + +--- + +### Task 8: Category ontology placement + +**Files:** +- Modify: `Model/lib/wdk/ontology/individuals.txt` + +`individuals.txt` is tab-delimited with 14 columns **and load-bearing empty fields**, including a trailing tab. Do not hand-type the row — derive it from the `VariationBySourceId` row so the whitespace is exact. + +- [ ] **Step 1: Look at the row you are copying** + +```bash +cd ~/workspaces/plasmodb/ApiCommonModel +grep -n "VariationQuestions.VariationBySourceId" Model/lib/wdk/ontology/individuals.txt | cat -A +``` + +Expected: one match (line ~1158) showing `^I` between every field, `topic_0199` as the parent, `search` as the target type, `menu` and `webservice` at the end, and a trailing `^I` before `$`. + +- [ ] **Step 2: Append the new row by substitution, not by typing** + +```bash +cd ~/workspaces/plasmodb/ApiCommonModel +grep "VariationQuestions.VariationBySourceId" Model/lib/wdk/ontology/individuals.txt \ + | sed 's/VariationBySourceId/VariationsByIsolateGroup/g' \ + >> Model/lib/wdk/ontology/individuals.txt +``` + +- [ ] **Step 3: Verify field count and content** + +```bash +cd ~/workspaces/plasmodb/ApiCommonModel +tail -n 1 Model/lib/wdk/ontology/individuals.txt | cat -A +tail -n 1 Model/lib/wdk/ontology/individuals.txt | awk -F'\t' '{print NF" fields"}' +grep -c "VariationQuestions.VariationsByIsolateGroup" Model/lib/wdk/ontology/individuals.txt +``` + +Expected: the new row with `VariationsByIsolateGroup` in columns 1 and 6 and everything else identical to the `VariationBySourceId` row; the same field count that row has (compare with `grep VariationBySourceId ... | awk -F'\t' '{print NF}'` — they must match); and `1`. + +- [ ] **Step 4: Build with `wb ontology` — NOT `wb model`** + +```bash +cd ~/workspaces/agentic-veupath-dev && bin/veup-build.sh plasmodb wb ontology +``` + +Expected: completes. `wb ontology` regenerates `individuals.owl` → `categories_merged.owl`, which is what the app actually reads, and does the model build too. Run `wb model` here instead and the site keeps serving the previous tree with **no error anywhere** — the search exists in the model but is uncategorized and absent from the menu. + +- [ ] **Step 5: Prove the OWL actually contains it** + +```bash +ssh cedar 'grep -c "VariationsByIsolateGroup" /var/www/jbrestel.plasmodb.org/gus_home/lib/wdk/ontology/categories_merged.owl' +``` + +Expected: non-zero. This is the server-side answer to "is it categorized where I intended" and needs no browser. + +- [ ] **Step 6: Commit** + +```bash +cd ~/workspaces/plasmodb/ApiCommonModel +git add Model/lib/wdk/ontology/individuals.txt +git commit -m "Categorize VariationsByIsolateGroup under Genetic Variation + +Same placement as VariationBySourceId: parent topic_0199, targetType search, +menu + webservice scopes. Requires wb ontology, not wb model -- a stale OWL +leaves the search uncategorized with no error anywhere." +``` + +--- + +### Task 9: End-to-end verification in the browser + +Everything in the params and queries is already verified by execution against the database. What only a live run can establish is the chain: hidden param → EDA queries → filter tree → plugin → results → record pages. **This is also the first real exercise of the `ApiCommonWebService` plumbing** (`/dnaseq` in the search dir, and the `variation_sample_meta` param name), which that repo's spec could not verify on its own. + +**Files:** none — this task changes nothing. + +- [ ] **Step 1: Mark the logs** + +```bash +cd ~/workspaces/agentic-veupath-dev && bin/veup-logs.sh plasmodb mark vbig +``` + +- [ ] **Step 2: Open the site and confirm which site you are on** + +Load `https://jbrestel.plasmodb.org/a/app` in Chrome and authenticate past the pre-release gate if prompted. Then, before trusting anything: + +```javascript +window.location.origin +``` + +Expected: `"https://jbrestel.plasmodb.org"`. If it reads `https://veupathdb.org`, **stop** — the tab bounced to autologin and every relative fetch from here answers for production. Authenticate and reload. + +- [ ] **Step 3: Confirm the search is registered for this site** + +From that authenticated page: + +```javascript +fetch('/a/service/record-types/variation') + .then(r => r.json()) + .then(d => d.searches.map(s => s.fullName).filter(n => n.includes('Variation'))) +``` + +Expected: includes `VariationQuestions.VariationsByIsolateGroup`. `/record-types` is project-filtered, so this — not the category tree — is the source of truth for whether this site has the search. + +If that 404s, the record type's URL segment is not `variation`; get the real one from `fetch('/a/service/record-types').then(r=>r.json()).then(d=>d.map(t=>t.urlSegment))` and retry. + +- [ ] **Step 4: Reach the search page and record its real URL** + +Navigate through the site's Searches menu (Genetic Variation → Differences Within a Group of Samples) rather than guessing a URL. **Record the URL you land on** in your report; later tasks and the other three searches will want it. + +- [ ] **Step 5: The organism param** + +Expected: a treeBox offering `Plasmodium falciparum 3D7` as the only selectable leaf, under branch nodes. If the tree is empty, the vocabulary query returned nothing for this project — re-run Task 2 Step 1. + +- [ ] **Step 6: The samples filter — the first proof the hidden param works** + +Select `Plasmodium falciparum 3D7`. Expected: the Samples filter populates with **216 samples** under a **7-category** tree (Provenance and identity, Organism under investigation, Specimen and culture, Collection event, Host, Collection location, Alignment statistics), with 20 leaf variables among them. + +This step exercises the hidden suffix param and both EDA queries end to end. Failure modes and what they mean: + +| symptom | cause | +|---|---| +| filter empty, no error | the suffix vocabulary returned zero rows — Task 3 Step 1 | +| `relation "eda.attributevalue_..." does not exist` | the suffix is wrong or the table is absent — Task 3 Step 2 | +| `Parent ontology ID 'sample' ... cannot be found` | the `CASE` on the parent is missing or wrong — Task 4 Step 2 | +| `The following ontology items have no children ... null item type` | the type mapping dropped a leaf's type — Task 4 Step 2 | + +- [ ] **Step 7: Run the search — the first exercise of the Java plumbing** + +Select at least two samples (accepting all 216 is fine), leave the thresholds at their defaults (80% RFT, minor allele frequency 0, percent called 20), and submit. + +Expected: a result page with rows, showing the `PercentMinorAlleles` and `PercentIsolateCalls` columns. + +| symptom | cause | +|---|---| +| `Organism dir does not exist` | the HSSS path — `buildNumber` (Task 1) or the plumbing's `/dnaseq` search dir | +| missing required parameter | the filterParam name does not match `getStrainFilterParamName()` (Task 4/6 Step 1) | +| `expected 4 columns` | the plugin's results-file contract — a plumbing problem, not a model one | + +Selecting exactly two samples is also worth one run: with `minSelectedCount="2"`, one sample must be rejected client-side. + +- [ ] **Step 8: Confirm the IDs resolve** + +Expected: IDs of the form `Variant__` (e.g. `Variant_Pf3D7_01_v3_100057`). Click one through to its record page and confirm it renders. This is the proof that the plumbing spec's ID-construction work holds through a real search — the plugin's `idPrefix` and `hsssReconstructSnpId` join must agree with what `VariationAttributes.source_id` actually contains. + +- [ ] **Step 9: Read the logs** + +```bash +cd ~/workspaces/agentic-veupath-dev && bin/veup-logs.sh plasmodb since vbig --quiet +``` + +Expected: the error logs report `silent:`. A healthy page load leaves them silent, so anything there is worth reading even if the page looked fine. + +- [ ] **Step 10: Report** + +Report to John: the search page URL, the sample and category counts you actually saw, the row count returned, one example variation ID that resolved to a record page, and the log verdict. If any step failed, report the symptom and the diagnosis from the tables above rather than guessing at a fix. + +--- + +## What this plan does not do + +Named so nobody thinks they were forgotten: + +- **The other three HSSS searches** — `ByLocation`, `ByGeneIds`, `ByTwoIsolateGroups`. Each reuses everything built in Tasks 2–5. +- **`FindMajorAllelesPlugin`'s param rename.** It hardcodes `ngsSnp_strain_meta_a` / `_m` as its own required-param contract. Deferred to the `ByTwoIsolateGroups` spec, which **must not forget it**. +- **Deleting the dead `snpParams.xml`** and the rest of the commented-out snp block. Once Task 5 copies the four params out, it has no remaining reason to exist — but removal has its own blast radius (`recordParams.xml`, `spanQuestions.xml`, `SnpsBySpanLogic`). +- **`snpParams.MinPercentMajorAlleles`, the `*Two` variants, and the wizard params** — used only by the two-group searches, so they migrate with `ByTwoIsolateGroups`. +- **Per-strain / VCF data.** `build-71/.../dnaseq/` also holds `vcf` and `bigwig` — inputs for the deferred strain tables on the variation record. +- **Reviving the HSSS test harnesses.** Both are broken (see the plumbing spec §4); this search's verification is the browser. From 46e06f405aab943132d07b260721b7120211fdb6 Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Wed, 5 Aug 2026 14:05:43 -0400 Subject: [PATCH 33/98] Point buildNumber at build-71 webservices files The HSSS variation files (/dnaseq/readFreq{20,40,60,80}) exist only under build-71; build-70 has no dnaseq directory at all. buildNumber is a model-wide constant feeding every webservices path, so this is its own commit: anyone bisecting a webservices-path problem needs to see it. Checked rather than assumed: build-71 is a superset of build-70 (1545 organism directories vs 1527) and is present for every ApiCommon project, so the searches already reading these files do not lose their inputs. Same motivation as 8ff801f03, which moved 68 -> 69. --- Model/lib/wdk/apiCommonModel.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Model/lib/wdk/apiCommonModel.xml b/Model/lib/wdk/apiCommonModel.xml index b63c1d166e..a39f87b352 100644 --- a/Model/lib/wdk/apiCommonModel.xml +++ b/Model/lib/wdk/apiCommonModel.xml @@ -25,7 +25,7 @@ 18 March 2026 12:09 1 January 1900 00:00 - 70 + 71 0 + + + + + + + + + + + + + + From 14871126b11fa3c27d846bccc1a29dcd095d5a5d Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Wed, 5 Aug 2026 14:10:45 -0400 Subject: [PATCH 35/98] Add hidden eda_sample_table_suffix param and VariationVQ querySet The organism param's internal value must be the taxon name -- the HSSS plugin resolves it through sres.TaxonName to build the webservices path. The EDA filter queries need the study+entity abbreviation to name per-study tables. Two identities for one dropdown, so the second travels in a hidden dependent param and gets interpolated into the table name. Looked up rather than recomputed from the SHA-1 convention: a lookup fails visibly (empty dropdown), a stale hash fails invisibly (missing relation, much later, from inside a filterParam). --- .../questions/params/variationParams.xml | 70 +++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/Model/lib/wdk/model/questions/params/variationParams.xml b/Model/lib/wdk/model/questions/params/variationParams.xml index 32d6fa2999..1eaddb21c6 100644 --- a/Model/lib/wdk/model/questions/params/variationParams.xml +++ b/Model/lib/wdk/model/questions/params/variationParams.xml @@ -22,6 +22,76 @@ default once their variation data loads. Omitted rather than invented. --> + + + + + + Derived from the selected organism. Not user-visible. + + + + + + + + + + + + + + + + + From 71bfa818b7c11c9bd8bbe0820cceb02c2df250ec Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Wed, 5 Aug 2026 14:14:19 -0400 Subject: [PATCH 36/98] Add EDA-driven variation_sample_meta filterParam and its two queries The samples filter moves from apidbTuning.Metadata to EDA's per-study tables, named by interpolating the hidden suffix param. The CASE on parent_stable_id is load-bearing, not defensive: EDA's attribute graph has no row for the entity itself, so seven category nodes point at a parent 'sample' that does not exist as a node, and WDK's validateOntologyItems throws on precisely that. Mapping unresolvable parents to NULL hands them to WDK's synthetic master root. Verified 27 nodes -> 7 roots, 0 dangling, 0 childless branches, 20 typed leaves. The param name is the plugin's required-parameter contract; do not rename it. --- .../questions/params/variationParams.xml | 107 ++++++++++++++++++ 1 file changed, 107 insertions(+) diff --git a/Model/lib/wdk/model/questions/params/variationParams.xml b/Model/lib/wdk/model/questions/params/variationParams.xml index 1eaddb21c6..c80fe028d0 100644 --- a/Model/lib/wdk/model/questions/params/variationParams.xml +++ b/Model/lib/wdk/model/questions/params/variationParams.xml @@ -43,6 +43,39 @@ Derived from the selected organism. Not user-visible. + + + + + + + Select a set of samples whose genomic sequences will be compared. Use the + sample characteristics to narrow the group, or accept all samples for the + organism you chose. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From e5946d4e15addb5b87f3b2c33101ac50a1adea40 Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Wed, 5 Aug 2026 14:15:58 -0400 Subject: [PATCH 37/98] Fix illegal -- inside XML comments in the plan's code blocks XML forbids a double hyphen inside . Three of the plan's nine XML blocks would not parse as dictated; the Task 4 implementer hit it and fixed the file, so the plan is what was stale. Every block now parses. --- .../plans/2026-08-05-variations-by-isolate-group.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/superpowers/plans/2026-08-05-variations-by-isolate-group.md b/docs/superpowers/plans/2026-08-05-variations-by-isolate-group.md index b2e2914b40..2f8ca919b5 100644 --- a/docs/superpowers/plans/2026-08-05-variations-by-isolate-group.md +++ b/docs/superpowers/plans/2026-08-05-variations-by-isolate-group.md @@ -511,7 +511,7 @@ After `EdaSampleTableSuffix`, inside the `VariationVQ` querySet: "Parent ontology ID 'sample' ... cannot be found" on exactly that. Mapping unresolvable parents to NULL makes WDK adopt them under its synthetic master root. Written as a membership test rather than - = 'sample' so it stays correct if the entity abbreviation differs -- + = 'sample' so it stays correct if the entity abbreviation differs - and it is already dynamic in the table name. 2. The type mapping is pinned to WDK's OntologyItemType enum, which accepts @@ -566,7 +566,7 @@ Inside `variationParams`, after `eda_sample_table_suffix`: params. Renaming it breaks the search at run time, not at build time. Internal values are EDA sample stable IDs, which are a strict subset of the - strain names in HSSS's strainIdToName.dat -- no mapping layer needed. The + strain names in HSSS's strainIdToName.dat - no mapping layer needed. The plugin writes them to a strains file and passes strains_are_names = 1. minSelectedCount=2 as the snp original: polymorphism within a group of one @@ -925,9 +925,9 @@ In `variationQuestions.xml`, inside the `VariationQuestions` questionSet, after + + + + + + + + + dflt + @WEBSERVICEMIRROR@/PROJECT_GOES_HERE/build-%%buildNumber%% + + + + + + + + This parameter applies to the sequencing reads of individual samples and + defines a stringency for data supporting a variant call between a sample and + the reference genome (Organism). Each nucleotide position of each sample is + compared to the reference genome and a call is made if the portion of the + sample's aligned reads that support the variant is above the Read Frequency + Threshold (RFT). Find high quality haploid variants with 80% RFT or + heterozygous diploid/aneuploid variants with 40%. See the Description below + for more. + + + + 80% + 80 + + + 60% + 60 + + + 40% + 40 + + + 20% + 20 + + + + + + + This parameter applies to your group of samples. A variant can occur in any + number of samples in your group and the least frequent call across all + samples is the Minor Allele Frequency. A variant will be returned by the + search if the frequency of the minor allele is equal to or greater than your + Minor Allele Frequency. See the Description below the Get Answer button for + more. + + + \d\d? + + + + + This parameter applies to the selected set of aligned sample sequences. At + any given nucleotide position, some samples in your group may not have data + supporting a base call because the Read Frequency Threshold was not met or + fewer than our minimum of 5 reads aligned. 'Percent samples with a base call' + defines the fraction of the selected samples that must have a base call + before a variant is returned for that nucleotide position, based on the + remaining samples that do have data. See the Description below for more + information. + + + \d\d?|100 + + + + + + + + + The Organism defines the species identity of the samples and the genome + against which each sample's variants were called. After choosing an Organism, + the set of samples available for forming groups is limited to samples aligned + to your chosen Organism's genome. + The organism you choose determines the genome to which the + variants have been mapped. It also restricts the set of samples you may + choose, since variants are identified by aligning that sample's reads to this + genome. + + + + + + + + + + + + + + + + From 13c85bfd37db945e923a8b24df535e2f85d76570 Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Wed, 5 Aug 2026 14:24:22 -0400 Subject: [PATCH 41/98] Fix the plugin source path in the plan (src/main/java, not lib/java) --- .../plans/2026-08-05-variations-by-isolate-group.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/superpowers/plans/2026-08-05-variations-by-isolate-group.md b/docs/superpowers/plans/2026-08-05-variations-by-isolate-group.md index 8c95be1ee4..03e478a4b3 100644 --- a/docs/superpowers/plans/2026-08-05-variations-by-isolate-group.md +++ b/docs/superpowers/plans/2026-08-05-variations-by-isolate-group.md @@ -822,8 +822,8 @@ from. Param names are unchanged -- the plugin reads them by name." ```bash cd ~/workspaces/plasmodb/ApiCommonWebService grep -rn "getStrainFilterParamName\|PARAM_ORGANISM\|REQUIRED" \ - WSFPlugin/lib/java/org/apidb/apicomplexa/wsfplugin/highspeedsnpsearch/FindPolymorphismsPlugin.java \ - WSFPlugin/lib/java/org/apidb/apicomplexa/wsfplugin/highspeedsnpsearch/FindPolymorphismsAbstractPlugin.java \ + WSFPlugin/src/main/java/org/apidb/apicomplexa/wsfplugin/highspeedsnpsearch/FindPolymorphismsPlugin.java \ + WSFPlugin/src/main/java/org/apidb/apicomplexa/wsfplugin/highspeedsnpsearch/FindPolymorphismsAbstractPlugin.java \ | head -20 ``` From 7ac703c18e9d315f0029de195ba7b8bc893d5309 Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Wed, 5 Aug 2026 14:30:02 -0400 Subject: [PATCH 42/98] Add the VariationsByIsolateGroup question Overrides attributesList, unlike VariationBySourceId: the plugin's dynamic minor-allele-frequency and percent-called columns are the point of the search and are not in the record's default summary. displayName says Samples rather than the snp original's Isolates, matching the EDA vocabulary the filter is built from. noSummaryOnSingleRecord deliberately unset -- a one-hit analytical result wants its context. Also adds a dynamicAttributes/columnAttribute block for PercentMinorAlleles, PercentIsolateCalls, and Phenotype -- the deprecated SNP equivalent (NgsSnpsByIsolateGroup) declares these the same way; without it, wb model fails with 'Summary attribute field [...] is invalid' because the wsColumns from the processQuery are not otherwise exposed as attribute fields the question can reference in attributesList. Co-Authored-By: Claude Opus 5 --- .../model/questions/variationQuestions.xml | 98 +++++++++++++++++++ 1 file changed, 98 insertions(+) diff --git a/Model/lib/wdk/model/questions/variationQuestions.xml b/Model/lib/wdk/model/questions/variationQuestions.xml index f814240746..9e7e6c2ba1 100644 --- a/Model/lib/wdk/model/questions/variationQuestions.xml +++ b/Model/lib/wdk/model/questions/variationQuestions.xml @@ -31,6 +31,104 @@ + + + + + + + + + + + + + +
+ + Each sample's sequencing reads are aligned to the reference genome + (Organism) and variants are recorded for each sample based on the Read + Frequency Threshold. Then, scanning variant locations across the group of + samples, variants are returned by the search if the Minor Allele Frequency + and the Percent samples with a base call are met. + +

Organism: The Organism parameter defines the species of the + samples and the genome in which the variants are determined. Choosing an + Organism focuses the Samples parameter to the samples of that organism, + changing the subset available when forming your group.

+ +

Samples: Sample sequences are accompanied by characteristics of + the sample -- where it was collected, the host, alignment statistics. By + default the group includes all samples from the Organism you chose; you may + narrow the group using those characteristics. At least two samples are + required, since polymorphism within a group of one is undefined.

+ +

Read frequency threshold: An allele is called for a sample at a + location if that fraction of the sample's aligned reads support it. For + example, a sample with 10 reads at a location -- 6 A and 4 C -- is called A + at a threshold of 60% or less, and not called at 80%. This matters most for + diploid or aneuploid organisms, where heterozygous positions are expected + near 50%.

+ +

Minor allele frequency: Among the qualifying calls at a location, + the minor allele frequency is the percent that are not the major allele. A + location is returned if that is at or above the value you specify. Use 0 to + find every variant location within the group.

+ +

Percent samples with a base call: A location is only considered if + this fraction of your selected samples have a qualifying call there. With 20 + samples and a threshold of 75%, a location with fewer than 15 called samples + is ignored.

+ ]]> + + + + + + Display the histogram of the values of this attribute + int + + + + + Display the histogram of the values of this attribute + int + + + + + + + From 06cf4032563b10f2a847e054d7772238711a9e15 Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Wed, 5 Aug 2026 14:31:41 -0400 Subject: [PATCH 43/98] Add the required dynamicAttributes block to the plan's Task 7 attributesList cannot name a processQuery's wsColumns until they are declared as dynamic attributes on the question; without the block, wb model fails with 'Summary attribute field [PercentMinorAlleles] ... is invalid'. The Task 7 implementer hit this, adapted the snp original's block, and verified the build. The three remaining HSSS variation searches will each need one. --- .../2026-08-05-variations-by-isolate-group.md | 28 ++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/docs/superpowers/plans/2026-08-05-variations-by-isolate-group.md b/docs/superpowers/plans/2026-08-05-variations-by-isolate-group.md index 03e478a4b3..48f6c58eef 100644 --- a/docs/superpowers/plans/2026-08-05-variations-by-isolate-group.md +++ b/docs/superpowers/plans/2026-08-05-variations-by-isolate-group.md @@ -999,10 +999,36 @@ In `variationQuestions.xml`, inside the `VariationQuestions` questionSet, after ]]> + + + + + Display the histogram of the values of this attribute + int + + + + + Display the histogram of the values of this attribute + int + + + + + ``` -- [ ] **Step 3: Verify the XML parses** +> **The `dynamicAttributes` block is mandatory.** An earlier version of this plan omitted it and the build failed with `Summary attribute field [PercentMinorAlleles] defined in question [...] is invalid` — `attributesList` may not reference a `processQuery`'s `wsColumn`s until they are declared as dynamic attributes on the question. The three remaining HSSS variation searches will each need their own block. ```bash python3 -c "import xml.etree.ElementTree as T; T.parse('$HOME/workspaces/plasmodb/ApiCommonModel/Model/lib/wdk/model/questions/variationQuestions.xml'); print('parses')" From cdd33e30946de5c3886aef72e6a7509855d4f1dc Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Wed, 5 Aug 2026 14:34:32 -0400 Subject: [PATCH 44/98] Categorize VariationsByIsolateGroup under Genetic Variation Same placement as VariationBySourceId: parent topic_0199, targetType search, menu + webservice scopes. Requires wb ontology, not wb model -- a stale OWL leaves the search uncategorized with no error anywhere. --- Model/lib/wdk/ontology/individuals.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/Model/lib/wdk/ontology/individuals.txt b/Model/lib/wdk/ontology/individuals.txt index a4746634b8..143c0964f6 100644 --- a/Model/lib/wdk/ontology/individuals.txt +++ b/Model/lib/wdk/ontology/individuals.txt @@ -1156,3 +1156,4 @@ VariationRecordClasses.VariationRecordClass.gene_count http://edamontology.org/t VariationRecordClasses.VariationRecordClass.TranscriptProducts http://edamontology.org/topic_0199 Genetic Variation VariationRecordClasses.VariationRecordClass table TranscriptProducts record download VariationRecordClasses.VariationRecordClass.PredictedEffects http://edamontology.org/topic_0199 Genetic Variation VariationRecordClasses.VariationRecordClass table PredictedEffects record download VariationRecordClasses.VariationRecordClass.VariationQuestions.VariationBySourceId http://edamontology.org/topic_0199 Genetic Variation VariationRecordClasses.VariationRecordClass search VariationQuestions.VariationBySourceId menu webservice +VariationRecordClasses.VariationRecordClass.VariationQuestions.VariationsByIsolateGroup http://edamontology.org/topic_0199 Genetic Variation VariationRecordClasses.VariationRecordClass search VariationQuestions.VariationsByIsolateGroup menu webservice From f5345b493365a0b4df00f4d3e46d60df0b45581a Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Wed, 5 Aug 2026 15:44:29 -0400 Subject: [PATCH 45/98] Design VariationsByLocation and VariationsByGeneIds Both reuse the ByIsolateGroup machinery wholesale; the only new model piece is a chromosome param, rewritten because the snp original's vocabulary reads a dead table and keys on org_abbrev while our organism param carries the taxon name. Its internal is the sequence source_id, which is what the HSSS files use. Records three more dead lookalikes that grep finds and the assembled model does not have, including the four ngsSnp_strain_meta_* filterParams sitting inside a commented-out region of sharedParams.xml -- which is why ByTwoIsolateGroups must build Set A/Set B fresh. Documents the emptyValue='No Match' coupling between sharedParams.sequenceId and the plugin's chromosome fallback: it reads as dead code from either side alone, and is ported unchanged rather than 'fixed'. No gene-to-variation join: FindSnpsByGeneIdsPlugin resolves gene IDs to intervals via webready.GeneAttributes_p and HSSS filters positionally, so apidb.VariationTranscriptProduct plays no part. searchCategory closed out: zero occurrences in the assembled model, so the previous spec's 'it arrives with ByLocation/ByGeneIds' resolves to never. --- ...iations-by-location-and-gene-ids-design.md | 337 ++++++++++++++++++ 1 file changed, 337 insertions(+) create mode 100644 docs/superpowers/specs/2026-08-05-variations-by-location-and-gene-ids-design.md diff --git a/docs/superpowers/specs/2026-08-05-variations-by-location-and-gene-ids-design.md b/docs/superpowers/specs/2026-08-05-variations-by-location-and-gene-ids-design.md new file mode 100644 index 0000000000..09165605fa --- /dev/null +++ b/docs/superpowers/specs/2026-08-05-variations-by-location-and-gene-ids-design.md @@ -0,0 +1,337 @@ +# `VariationsByLocation` and `VariationsByGeneIds` — design + +**Date:** 2026-08-05 +**Status:** approved +**Scope:** Two searches on the `variation` record, ported from the deprecated `snp` record's +`NgsSnpsByLocation` and `NgsSnpsByGeneIds`. Both are HSSS `processQuery`s that reuse the +param machinery built for `VariationsByIsolateGroup` and add one region restriction each. +**Implementation target:** `ApiCommonModel`, plus a one-constant change in +`ApiCommonWebService` (§5). Branch `dnaseq-merge-experiments`. +**Prerequisites, both implemented:** +- `2026-08-05-hsss-variation-plumbing-design.md` (`ApiCommonWebService`) +- `2026-08-05-variations-by-isolate-group-design.md` — this spec reuses its params + wholesale. Section references prefixed `group §` point there; `plumbing §` to the former. + +## 1. Purpose + +`VariationsByIsolateGroup` answers "what differs within this group of samples, genome-wide." +These two restrict the same computation to a region of interest: + +- **`VariationsByLocation`** — a chromosome or sequence ID, with optional start/end. +- **`VariationsByGeneIds`** — a gene list, resolved to the genes' genomic intervals. + +They are the cheapest two of the three remaining ports, because the expensive part — +the organism vocabulary, the hidden EDA table-suffix param, and the EDA-driven samples +filter — already exists and is verified working end to end (group §8.2, confirmed live: +216 samples, 7 categories, 506,772 results, `Variant_*` IDs resolving to record pages). + +Both plugins extend `FindPolymorphismsPlugin`, so they inherit +`getStrainFilterParamName() → variation_sample_meta` and need no further contract work +beyond §5. + +## 2. Reused unchanged — the whole point of doing `ByIsolateGroup` first + +| Component | Where | +|---|---| +| `organismVQ.withVariationsTree` | `organismParams.xml`, group §4.1 | +| `variationParams.eda_sample_table_suffix` + `VariationVQ.EdaSampleTableSuffix` | group §4.2 | +| `variationParams.variation_sample_meta` + `VariationVQ.SamplesMetadataByStudy` / `SampleOntologyByStudy` | group §4.3, §5 | +| `variationParams.WebServicesPath`, `ReadFrequencyPercent`, `MinPercentMinorAlleles`, `MinPercentIsolateCalls` | group §4.4 | +| `sharedParams.sequenceId`, `start_point`, `end_point` | live; `ByLocation` only | +| `sharedParams.ds_gene_ids` | live; `ByGeneIds` only | +| The five `wsColumn`s and the `dynamicAttributes` block | plugin-dictated, group §7.1 | + +The four `sharedParams` entries were each confirmed present in the **assembled** model +(`wdkXml -model PlasmoDB`), not merely in the file — see §3 for why that distinction is +load-bearing here. + +`ds_gene_ids` is a `datasetParam` with `recordClassRef="TranscriptRecordClasses.TranscriptRecordClass"`. +It is generic (any gene list) and needs no variation-specific replacement. + +## 3. Three dead lookalikes (`grep` finds them; the model does not have them) + +This is the same trap that cost a correction in the previous spec, and it appears three more +times. In every case the source file contains the name and the assembled model does not. + +| Name | Why it is unusable | +|---|---| +| `sharedParams.ngsSnp_strain_meta_a` / `_m` / `_a_wiz` / `_m_wiz` | **Inside a commented-out region** of `sharedParams.xml`. Parses as XML comments; absent from the model. (Their `metadataQueryRef`s point at `SnpVQ.*`, itself absent — see group §4.4.) Affects `ByTwoIsolateGroups`, not this spec, but recorded here because the discovery belongs with the others. | +| `organismVQ.withNgsSNPs` | Live XML, but reads `apidbtuning.snpstrains`, **which does not exist in this build** — the same dead table that forced the `withVariationsTree` rewrite. | +| `sharedParams.chromosomeOptionalForNgsSnps` | Live *and* in the assembled model, so it looks reusable. It is not: its vocabulary `SharedVQ.ChromosomeForNgsSnps` overrides the organism param to the dead `organismVQ.withNgsSNPs`, **and** its SQL keys on `org_abbrev = '$$organismSinglePick$$'` while our organism param deliberately carries the **taxon name** (group §2). Even against a live table that comparison matches nothing. | + +> **Verify absence with the bare name, never `name="x"`.** `wdkXml` prints attributes +> single-quoted (`name='x'`), so a double-quoted grep pattern reports zero regardless of what +> the model contains — a check that can only pass. Print the full paramSet list alongside. + +## 4. The chromosome param — new, rewritten + +### 4.1 `variationParams.chromosomeOptionalForVariations` + +```xml + +``` + +Shape carried over from the snp original. It lives in `variationParams` rather than +`sharedParams` because only variation searches use it, and because §5 renames the plugin's +constant to match. + +### 4.2 `VariationVQ.ChromosomeForVariations` + +```sql +SELECT * FROM ( + SELECT DISTINCT s.chromosome AS term, + s.source_id AS internal, + s.chromosome_order_num + FROM webready.GenomicSeqAttributes_p s + WHERE s.organism = '$$organismSinglePick$$' + AND s.chromosome IS NOT NULL + UNION + SELECT 'Choose chromosome' AS term, 'choose' AS internal, -1 AS chromosome_order_num +) t +ORDER BY chromosome_order_num +``` + +Two changes from `SharedVQ.ChromosomeForNgsSnps`, both forced: + +- **`s.organism`, not `s.org_abbrev`.** The organism param's internal value is the scientific + taxon name, because `HighSpeedSnpSearchAbstractPlugin` resolves it through `sres.TaxonName` + to get `name_for_filenames` for the webservices path (group §2). `GenomicSeqAttributes_p` + carries both columns, so this is a one-word change. +- **No `queryRef` override on a depended organism param.** The original pointed at the dead + `organismVQ.withNgsSNPs`; the paramRef in each `processQuery` supplies + `organismVQ.withVariationsTree` instead (§6). + +Verified 2026-08-05 against `unidb_shu_a`: for `Plasmodium falciparum 3D7` this returns +**15 rows in 26 ms** — the 14 chromosomes, `term` `01` through `14` in `chromosome_order_num` +order, plus the sentinel. + +**The `internal` is the sequence source ID, and that is what makes it work:** chromosome `01` +yields `Pf3D7_01_v3`, which is exactly the sequence identifier the HSSS files carry — the +live `ByIsolateGroup` run returned `Variant_Pf3D7_01_v3_1`. The value feeds the position +filter directly with no mapping. + +`webready.GenomicSeqAttributes_p` is `LIST`-partitioned on `org_abbrev`, so filtering on +`organism` forfeits partition pruning. Measured at 25 ms, which is well inside what a +vocabulary query needs, so the simpler predicate wins over a join to recover the abbreviation. +If it ever becomes hot, the fix is to resolve `org_abbrev` from the taxon name in a subquery — +not to change what the organism param carries. + +The `'Choose chromosome'/'choose'` sentinel row is carried over deliberately; see §4.3. + +### 4.3 The empty-sequence contract — port it faithfully, and do not "fix" it + +`FindPolymorphismsWithSeqFilterPlugin.makeCommandToCreateBashScript` reads: + +```java +String seq = params.get(PARAM_SEQUENCE); +if (seq.contains("No Match")) seq = chromosome; +``` + +This looks like dead code — `sequenceId` has no default value, so a blank box would appear to +leave `seq` empty and the fallback unreachable, making the chromosome dropdown decorative. +**It is not dead.** `sharedParams.sequenceId` carries twelve per-project `` children, +each with: + +```xml + +``` + +WDK substitutes the literal string `No Match` for an empty value, which is precisely what the +guard tests. Sequence ID takes precedence; the chromosome dropdown is the fallback when the +box is left empty. The coupling is invisible from either side alone — the Java names a magic +string, the model supplies it from an attribute on a child element — which is why it is +recorded here. + +Consequences for this port, all deliberate: + +- Keep `seq.contains("No Match")` exactly as is. Widening it to also treat blank/null as empty + was considered and **rejected**: it would be a behavior change to a plugin shared with the + chip-snp searches, in service of a case that cannot arise while `emptyValue` is set. +- Keep the `'choose'` sentinel in the vocabulary, so "no chromosome chosen" is a real value + rather than an empty param. +- `end_point` of `0` means "to the end of the sequence" — the plugin maps it to `1000000000`. + Carried over as-is; the param's prompt already says `End Location (0 = end)`. + +## 5. `ApiCommonWebService`: one constant + +```java +- public static final String PARAM_CHROMOSOME = "chromosomeOptionalForNgsSnps"; ++ public static final String PARAM_CHROMOSOME = "chromosomeOptionalForVariations"; +``` + +in `FindPolymorphismsWithSeqFilterPlugin`. Nothing else in that class changes. + +Why rename at all, when overriding the existing param's `queryRef` would have avoided a +cross-repo change: the strain filter was already renamed to `variation_sample_meta` +(plumbing §3.5). Leaving the chromosome name alone would produce a single `processQuery` +declaring `variation_sample_meta` and `chromosomeOptionalForNgsSnps` side by side — a +half-renamed contract, which is worse than either uniform choice and invites the next reader +to re-derive that the snp-era name is meaningless. The plugin is variation-only from here. + +Requires `bld ApiCommonWebService` before the model change can be exercised. `PARAM_SEQUENCE`, +`PARAM_START_POINT`, and `PARAM_END_POINT` are already generic and keep their names. + +## 6. The two searches + +Both `processQuery`s share this prefix — organism param with the variation vocabulary, then +the hidden suffix param, then the samples filter, then the four thresholds: + +```xml + + +... + + + + + +``` + +`quote="false"` on the organism param is required for the dependent-param query; the plugin +strips quotes for its own use. + +### 6.1 `VariationsByLocation` + +`processName="...highspeedsnpsearch.FindPolymorphismsWithSeqFilterPlugin"`, inserting after +the suffix param: + +```xml + + + + +``` + +Question `displayName="Genomic Location"`, `shortDisplayName="Location"`. + +### 6.2 `VariationsByGeneIds` + +`processName="...highspeedsnpsearch.FindSnpsByGeneIdsPlugin"`, inserting: + +```xml + + + +``` + +The per-project `default` on the paramRef is the snp original's pattern. Defaults are supplied +only for the three projects with variation data loaded today; `AmoebaDB`, `CryptoDB`, +`MicrosporidiaDB`, `PiroplasmaDB`, and `ToxoDB` owe one when their data lands — omitted rather +than invented, matching the `variation_id` param (searches §4). + +Question `displayName="Gene ID(s)"`, `shortDisplayName="Gene IDs"`. + +**No gene→variation join is needed anywhere in the model.** `FindSnpsByGeneIdsPlugin` +resolves the gene list to genomic intervals itself: + +```sql +select g.sequence_id, g.start_min, g.end_max +from webready.GeneAttributes_p g, () user_genes +where g.source_id = user_genes.gene_source_id +``` + +writes them to `genomicLocations.txt`, and HSSS filters variant positions by interval via +`hsssGenomicLocationsFilter`. `webready.geneattributes_p` exists in this build (verified), so +that SQL is untouched. The gene↔variation relationship is **positional**, not product-based: +`apidb.VariationTranscriptProduct` (4.6M rows, keyed `sequence_source_id, location, +na_feature_id`) is real and populated but plays no part in this path. Reporting per-gene coding +consequences would be a scope increase beyond the snp original and is explicitly not in this +spec (§9). + +### 6.3 Both questions + +Same `attributesList` as `ByIsolateGroup`: + +```xml + +``` + +and each needs **its own `dynamicAttributes` block** declaring `PercentMinorAlleles`, +`PercentIsolateCalls`, and `Phenotype` — copied from `VariationsByIsolateGroup`. + +> **`dynamicAttributes` is mandatory, not decoration.** `attributesList` may not name a +> `processQuery`'s `wsColumn`s until they are declared on the question; without the block the +> build fails with `Summary attribute field [PercentMinorAlleles] defined in question [...] is +> invalid`. This cost a build cycle on the previous search. + +`noSummaryOnSingleRecord` is not set, for the same reason as `ByIsolateGroup`: a one-hit +analytical result wants its context. + +## 7. Ontology, and the `searchCategory` question closed + +Two rows in `Model/lib/wdk/ontology/individuals.txt`, parent +`http://edamontology.org/topic_0199`, `targetType` `search`, scopes `menu` + `webservice` — +identical placement to the two existing variation searches. Derive each row by substitution +from the `VariationBySourceId` row rather than typing it: the file is tab-delimited with +load-bearing empty fields and a trailing tab. + +**No `searchCategory` on either question, and the previous spec's "it arrives with +ByLocation/ByGeneIds" resolves to "it never arrives."** `searchCategory` appears **zero times** +in the 17,763-line `wdkXml` dump of the assembled model — it is legacy and is not surfaced by +the service. Menu grouping comes from the category ontology, which these rows handle. + +Build both with **`wb ontology`, not `wb model`** — a categorization change leaves the OWL +stale with no error anywhere. + +## 8. Verification + +Everything in §4.2 is already verified by execution. What a live run must establish, per search: + +1. `wb ontology` completes; `/service/record-types/variation` lists the new search + (project-filtered, so this is the source of truth for presence). +2. **`ByLocation`:** the chromosome dropdown offers `01`–`14`; choosing `01` and submitting with + the sequence box left empty returns rows confined to `Pf3D7_01_v3`. That last clause is the + real test — it exercises the `emptyValue`→`No Match`→chromosome-fallback chain of §4.3 end to + end. Then repeat with an explicit `sequenceId` and a start/end window and confirm the window + is respected. +3. **`ByGeneIds`:** the PlasmoDB default `PF3D7_1133400` returns rows whose locations fall inside + that gene's `start_min`/`end_max` (check against `webready.GeneAttributes_p`). **This is the + first real exercise of `hsssGenomicLocationsFilter`**, whose ID join was fixed in the + plumbing *plan*'s Task 1b (a second composition site found during implementation, so it is + in the plan rather than that spec) — the installed copy in `gus_home` was confirmed to carry + the fix (`underscore=2, dotted=0`). A wrong ID form here surfaces as *zero results with no + error*, so a non-empty result set is the assertion. +4. Returned IDs are `Variant__` and resolve to real record pages. +5. `veup-logs.sh plasmodb mark/since` around each: error logs silent. + +Fetches in step 1 must come from an authenticated app page; a raw `curl` redirects to autologin, +and a tab bounced to that gate makes relative `fetch` calls answer for **veupathdb.org +production**. Check `window.location.origin` first. Note the app path redirects to the webapp +context — on this instance `/plasmo.jbrestel/app`, service at `/plasmo.jbrestel/service` — so +build relative paths from the post-redirect location, not from `/a/`. + +Screenshots were unavailable during the previous search's verification (the Chrome extension's +script injection times out on this instance, reproducibly, including on known-good pages). +`javascript_tool` and the service endpoints work; plan verification around them. + +## 9. Out of scope + +- **`VariationsByTwoIsolateGroups`.** Its own spec: a different plugin + (`FindMajorAllelesPlugin`), an 11-column `wsColumn` set, five doubled threshold params, and + **two new EDA-driven filter params** — `ngsSnp_strain_meta_a`/`_m` cannot be reused (§3), so + Set A and Set B must be built fresh from `variation_sample_meta`'s shape. That spec **must + not forget** `FindMajorAllelesPlugin`'s hardcoded `ngsSnp_strain_meta_a` / `_m` names, and + should decide deliberately what the odd `_m` becomes given its prompts read "Set B". +- **`NgsSnpsByTwoIsolateGroupsWiz`** — a sixth search (PlasmoDB/UniDB only) using the + `*_wiz` params. Decide with `ByTwoIsolateGroups` whether it is worth porting at all. +- **Per-gene coding consequences** from `apidb.VariationTranscriptProduct` (§6.2). A feature, + not a port. +- **Deleting the dead `snpParams.xml`** and the commented-out snp regions of `sharedParams.xml`. + Own change, own blast radius. +- **`ReadFrequencyPercent` as a functional parameter.** Investigated 2026-08-05: for haploid + organisms the upstream caller runs `freebayes --min-alternate-fraction 0.8` + (`dnaseq-nextflow/modules/snp.nf:25`), so every variant reaching + `processSequenceVariations.jl`'s cutoff ladder `(20,40,60,80)` already passes 80, and all four + `readFreq*` directories receive identical writes — confirmed on disk (equal byte sizes, + matching `md5sum`, distinct inodes). The param correctly selects its directory; the + directories hold the same data. **Any QA that tries to verify "changing read frequency changes + results" will get a false negative on a haploid site**, which is worth knowing before it is + reported as a bug in these searches. Not ours to fix. From 7b8501d7134e2a2e23fc35356296a6d68736c45a Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Wed, 5 Aug 2026 15:51:44 -0400 Subject: [PATCH 46/98] Add implementation plan for VariationsByLocation and VariationsByGeneIds Six tasks: the ApiCommonWebService constant rename (with an installed-jar check, since a stale jar makes the model look wrong), the chromosome param and its vocabulary, one task per search (query + question + dynamicAttributes + build), both ontology rows under one wb ontology, then browser verification. Verification asserts on locations rather than row counts: the by-location run with an empty sequence box must return only Pf3D7_01_v3 IDs (the emptyValue -> No Match -> chromosome fallback), and the by-gene run must land inside PF3D7_1133400's span 1292966-1296696, which is the first real exercise of hsssGenomicLocationsFilter's ID join. Every XML block parser-checked; all param and query references resolve to something this plan defines or that already exists. --- ...-05-variations-by-location-and-gene-ids.md | 962 ++++++++++++++++++ 1 file changed, 962 insertions(+) create mode 100644 docs/superpowers/plans/2026-08-05-variations-by-location-and-gene-ids.md diff --git a/docs/superpowers/plans/2026-08-05-variations-by-location-and-gene-ids.md b/docs/superpowers/plans/2026-08-05-variations-by-location-and-gene-ids.md new file mode 100644 index 0000000000..33939c35a4 --- /dev/null +++ b/docs/superpowers/plans/2026-08-05-variations-by-location-and-gene-ids.md @@ -0,0 +1,962 @@ +# `VariationsByLocation` + `VariationsByGeneIds` Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Add the two region-restricted HSSS variation searches — by genomic location and by gene list — reusing the param machinery already built for `VariationsByIsolateGroup`. + +**Architecture:** One constant renamed in `ApiCommonWebService`; in `ApiCommonModel`, one new chromosome param plus its vocabulary query, two `processQuery`s, two questions, two category-ontology rows. Everything else is reused verbatim. The design is `docs/superpowers/specs/2026-08-05-variations-by-location-and-gene-ids-design.md` — read it; this plan implements it and does not restate its reasoning. + +**Tech Stack:** WDK model XML; Java (one string constant); PostgreSQL (`unidb_shu_a`); the `agentic-veupath-dev` control plane for remote builds on `cedar`; Claude in Chrome for verification. + +--- + +## Orientation + +Read this before Task 1 even if you implemented the previous search — two items are new. + +**Repos.** This work spans two, both on branch **`dnaseq-merge-experiments`**, never `main`: + +| | | +|---|---| +| `~/workspaces/plasmodb/ApiCommonWebService` | Task 1 only (one Java constant) | +| `~/workspaces/plasmodb/ApiCommonModel` | Tasks 2–5 | +| `~/workspaces/agentic-veupath-dev` | control plane — run `bin/veup-*.sh` from **here** | + +Local edits reach `cedar` through a running `mutagen` sync. You never copy files. Builds run remotely. + +**There is no unit-test framework for WDK model XML.** Don't look for one. Verification is `psql` for SQL, a one-second `ElementTree` parse for well-formedness, a remote build for reference resolution, and `wdkXml` to prove presence in the *assembled* model rather than in a file. + +```bash +psql -h localhost -p 5432 -d unidb_shu_a # read-only; every query here is a SELECT +``` + +> **Never** `INSERT`/`UPDATE`/`DELETE`/`ALTER` or touch an index in any schema but `jbrestel`. + +**Five traps, all of which have already cost time on this feature:** + +1. **Flags go BEFORE the profile name.** `bin/veup-build.sh plasmodb wb model --dry-run` silently drops the flag **and runs for real**. A real dry run prints `DRYRUN:`-prefixed lines. +2. **XML forbids `--` inside ``.** Every comment below is checked. If you reword one, keep double hyphens out or the file will not parse. Text inside `` is exempt. +3. **`wdkXml` prints attributes single-quoted** (`name='x'`). A double-quoted grep pattern matches nothing regardless of what the model contains. +4. **`dynamicAttributes` is mandatory** on any question whose `attributesList` names a `processQuery`'s `wsColumn`s. Both questions here need one. Omitting it fails the build with `Summary attribute field [PercentMinorAlleles] ... is invalid`. +5. **A remote grep for a `$`-containing pattern gets expanded by the remote shell** unless single-quoted on the remote side. Use `ssh host "... '\$foo' ..."`. + +**Do not "fix" the `No Match` guard.** `FindPolymorphismsWithSeqFilterPlugin` contains `if (seq.contains("No Match")) seq = chromosome;`, which looks like dead code. It is live: `sharedParams.sequenceId` has twelve per-project `` children each carrying `allowEmpty="true" emptyValue="No Match"`, and WDK substitutes that literal for an empty box. Design §4.3. Leave it exactly as it is. + +--- + +## File Structure + +| File | Change | Responsibility | +|---|---|---| +| `ApiCommonWebService/.../FindPolymorphismsWithSeqFilterPlugin.java:17` | Modify | the chromosome param name contract (Task 1) | +| `ApiCommonModel/Model/lib/wdk/model/questions/params/variationParams.xml` | Add 1 query to `VariationVQ`, 1 param to `variationParams` | the chromosome param (Task 2) | +| `ApiCommonModel/Model/lib/wdk/model/questions/queries/variationQueries.xml` | Add 2 `processQuery`s | plugin bindings (Tasks 3, 4) | +| `ApiCommonModel/Model/lib/wdk/model/questions/variationQuestions.xml` | Add 2 questions | the user-facing searches (Tasks 3, 4) | +| `ApiCommonModel/Model/lib/wdk/ontology/individuals.txt` | Append 2 rows | category placement (Task 5) | + +--- + +### Task 1: Rename the chromosome param constant in `ApiCommonWebService` + +**Files:** +- Modify: `WSFPlugin/src/main/java/org/apidb/apicomplexa/wsfplugin/highspeedsnpsearch/FindPolymorphismsWithSeqFilterPlugin.java:17` + +- [ ] **Step 1: Confirm you are changing exactly one of the two `PARAM_CHROMOSOME` declarations** + +```bash +cd ~/workspaces/plasmodb/ApiCommonWebService +git branch --show-current +grep -rn "PARAM_CHROMOSOME =" WSFPlugin/src/main/java/org/apidb/apicomplexa/wsfplugin/highspeedsnpsearch/ +``` + +Expected: branch `dnaseq-merge-experiments`, and exactly two declarations: + +``` +FindChipPolymorphismsWithSeqFilterPlugin.java:20: ... PARAM_CHROMOSOME = "chromosomeOptional"; +FindPolymorphismsWithSeqFilterPlugin.java:17: ... PARAM_CHROMOSOME = "chromosomeOptionalForNgsSnps"; +``` + +**Only the second changes.** The chip plugin uses `chromosomeOptional`, a different param serving live chip-snp searches; touching it would break them. + +- [ ] **Step 2: Make the change** + +`FindPolymorphismsWithSeqFilterPlugin.java` line 17: + +```java + public static final String PARAM_CHROMOSOME = "chromosomeOptionalForVariations"; +``` + +Change nothing else in the file. In particular leave line 44's +`if (seq.contains("No Match")) seq = chromosome;` untouched — see Orientation. + +- [ ] **Step 3: Verify the diff is one line** + +```bash +cd ~/workspaces/plasmodb/ApiCommonWebService && git diff --stat && git diff +``` + +Expected: `1 file changed, 1 insertion(+), 1 deletion(-)`, showing only the string literal change. + +- [ ] **Step 4: Commit** + +```bash +cd ~/workspaces/plasmodb/ApiCommonWebService +git add WSFPlugin/src/main/java/org/apidb/apicomplexa/wsfplugin/highspeedsnpsearch/FindPolymorphismsWithSeqFilterPlugin.java +git commit -m "Rename the chromosome param contract for variation searches + +FindPolymorphismsWithSeqFilterPlugin serves VariationsByLocation now. Its +sibling contract, the strain filter, was already renamed to +variation_sample_meta; leaving this one as chromosomeOptionalForNgsSnps would +give a single processQuery two differently-named eras of the same plugin and +invite the next reader to re-derive that the snp name is meaningless. + +The chip plugin's own PARAM_CHROMOSOME (chromosomeOptional) is untouched: it +serves live chip-snp searches." +``` + +- [ ] **Step 5: Build and install** + +```bash +cd ~/workspaces/agentic-veupath-dev && \ + ssh -o LogLevel=ERROR "$(python3 bin/resolve.py --profile profiles/plasmodb.yml --field host)" \ + "bash -lc 'source /var/www/jbrestel.plasmodb.org/etc/setenv && bld ApiCommonWebService'" +``` + +Expected: `BUILD SUCCESSFUL` (roughly 1–2 minutes). `Test-Installation` is not in the default +depends list, so the non-compiling JUnit module is not built. + +- [ ] **Step 6: Reload, and confirm the new name reached the installed jar** + +WSF plugins are loaded by the webapp; the constant does not take effect until a reload. + +```bash +cd ~/workspaces/agentic-veupath-dev && bin/veup-build.sh plasmodb reload +``` + +Expected: `OK - Reloaded application at context path [/plasmo.jbrestel]`. + +```bash +ssh cedar "bash -lc 'cd /var/www/PlasmoDB/plasmo.jbrestel/webapp/WEB-INF/lib && \ + for j in *.jar; do unzip -p \$j org/apidb/apicomplexa/wsfplugin/highspeedsnpsearch/FindPolymorphismsWithSeqFilterPlugin.class 2>/dev/null | \ + strings | grep -l chromosomeOptionalForVariations >/dev/null && echo \$j; done'" 2>/dev/null +``` + +Expected: at least one jar name printed. If nothing prints, try the simpler form below; the +point is to confirm the *installed* class carries the new string, not just the source tree. + +```bash +ssh cedar "bash -lc 'grep -rl chromosomeOptionalForVariations /var/www/PlasmoDB/plasmo.jbrestel/webapp/WEB-INF/lib/ 2>/dev/null | head'" +``` + +Expected: one or more jar paths. **A pass here is what makes Task 6 meaningful** — with the old +string installed, the search fails at run time with a missing-required-parameter error and the +model looks wrong when it is not. + +--- + +### Task 2: The chromosome param and its vocabulary + +**Files:** +- Modify: `ApiCommonModel/Model/lib/wdk/model/questions/params/variationParams.xml` + +- [ ] **Step 1: Run the vocabulary SQL — this is the test** + +The model substitutes the organism param at run time; run it with the value inlined: + +```bash +psql -h localhost -p 5432 -d unidb_shu_a -c " +SELECT * FROM ( + SELECT DISTINCT s.chromosome AS term, + s.source_id AS internal, + s.chromosome_order_num + FROM webready.GenomicSeqAttributes_p s + WHERE s.organism = 'Plasmodium falciparum 3D7' + AND s.chromosome IS NOT NULL + UNION + SELECT 'Choose chromosome' AS term, 'choose' AS internal, -1 AS chromosome_order_num +) t +ORDER BY chromosome_order_num" +``` + +Expected: **15 rows** — the sentinel first (`chromosome_order_num = -1`), then `01` through `14` +with internals `Pf3D7_01_v3` … `Pf3D7_14_v3`. + +The `internal` values are the load-bearing part: `Pf3D7_01_v3` is exactly the sequence +identifier the HSSS files carry (the live `ByIsolateGroup` run returned +`Variant_Pf3D7_01_v3_1`), so the value feeds the position filter with no mapping. + +- [ ] **Step 2: Confirm why the predicate is `organism` and not `org_abbrev`** + +```bash +psql -h localhost -p 5432 -d unidb_shu_a -c " +SELECT count(*) FROM webready.GenomicSeqAttributes_p +WHERE org_abbrev = 'Plasmodium falciparum 3D7'" +``` + +Expected: `0`. The snp original keyed on `org_abbrev`, but our organism param carries the +**taxon name** because the HSSS plugin resolves it through `sres.TaxonName` to build the +webservices path. That zero is why the column changed. + +- [ ] **Step 3: Add the vocabulary query** + +In `variationParams.xml`, inside the existing `VariationVQ` querySet, after +`SampleOntologyByStudy`: + +```xml + + + + + + + + + +``` + +`noTranslation="true"` passes the organism param's **term** rather than its internal, matching +`EdaSampleTableSuffix` and the snp-era precedent. The SQL supplies the quotes. + +Note the query declares only `internal` and `term` as columns even though the SQL selects +`chromosome_order_num` — that third column exists solely to drive `ORDER BY`, and the snp +original did the same. + +- [ ] **Step 4: Add the param** + +Inside the `variationParams` paramSet, after `MinPercentIsolateCalls`: + +```xml + + + + + + + + + +``` + +- [ ] **Step 5: Verify the XML parses and the pieces landed in the right containers** + +```bash +cd ~/workspaces/plasmodb/ApiCommonModel && python3 - <<'PY' +import xml.etree.ElementTree as T +r = T.parse('Model/lib/wdk/model/questions/params/variationParams.xml').getroot() +for ps in r.findall('paramSet'): + print('paramSet', ps.get('name'), [(c.tag, c.get('name')) for c in ps]) +for qs in r.findall('querySet'): + print('querySet', qs.get('name'), [q.get('name') for q in qs.findall('sqlQuery')]) +PY +``` + +Expected: `variationParams` now lists `variation_id`, `eda_sample_table_suffix`, +`variation_sample_meta`, `WebServicesPath`, `ReadFrequencyPercent`, `MinPercentMinorAlleles`, +`MinPercentIsolateCalls`, `chromosomeOptionalForVariations`; and `VariationVQ` lists +`EdaSampleTableSuffix`, `SamplesMetadataByStudy`, `SampleOntologyByStudy`, +`ChromosomeForVariations`. + +- [ ] **Step 6: Commit** + +```bash +cd ~/workspaces/plasmodb/ApiCommonModel +git add Model/lib/wdk/model/questions/params/variationParams.xml +git commit -m "Add chromosomeOptionalForVariations param and its vocabulary + +Rewritten, not reused. The snp original's vocabulary reaches +organismVQ.withNgsSNPs, which reads the nonexistent apidbtuning.snpstrains, +and keys on org_abbrev while our organism param carries the taxon name (0 rows +if you try it). Keyed on organism instead: 15 rows in 26ms. + +internal is the sequence source_id (chromosome 01 gives Pf3D7_01_v3), which is +the identifier the HSSS files use, so the value feeds the position filter with +no mapping layer." +``` + +--- + +### Task 3: `VariationsByLocation` — query, question, build + +**Files:** +- Modify: `ApiCommonModel/Model/lib/wdk/model/questions/queries/variationQueries.xml` +- Modify: `ApiCommonModel/Model/lib/wdk/model/questions/variationQuestions.xml` + +- [ ] **Step 1: Confirm the plugin's required params before wiring to them** + +```bash +cd ~/workspaces/plasmodb/ApiCommonWebService +sed -n '15,35p' WSFPlugin/src/main/java/org/apidb/apicomplexa/wsfplugin/highspeedsnpsearch/FindPolymorphismsWithSeqFilterPlugin.java +``` + +Expected: `PARAM_CHROMOSOME = "chromosomeOptionalForVariations"` (Task 1's change), and +`getExtraParamNames()` returning all four of chromosome, sequence, start point, end point. +All four are **required** — none may be omitted from the `processQuery`. + +- [ ] **Step 2: Add the process query** + +In `variationQueries.xml`, inside the `VariationsBy` querySet, after +`VariationsByIsolateGroup`: + +```xml + + + + + + + + The Organism defines the species identity of the samples and the genome + against which each sample's variants were called. + + + + + + + + + + + + + + + + + + + +``` + +- [ ] **Step 3: Add the question** + +In `variationQuestions.xml`, inside the `VariationQuestions` questionSet, after +`VariationsByIsolateGroup`: + +```xml + + + + + + + + + + + + +
+ + Each sample's sequencing reads are aligned to the reference genome (Organism) + and variants are recorded for each sample based on the Read Frequency + Threshold. Then, scanning variant locations within your region across the + group of samples, variants are returned if the Minor Allele Frequency and the + Percent samples with a base call are met. + +

Defining the region: Either choose a Chromosome, or enter a Genomic + sequence ID. A sequence ID you enter takes precedence; the Chromosome menu is + used when you leave the sequence box empty. Start and End restrict the region + further, and an End of 0 means "to the end of the sequence".

+ +

Organism: The Organism parameter defines the species of the samples + and the genome in which the variants are determined. Choosing an Organism + focuses the Samples parameter to the samples of that organism.

+ +

Samples: By default the group includes all samples from the Organism + you chose; you may narrow it using the sample characteristics. At least two + samples are required, since polymorphism within a group of one is undefined.

+ +

Read frequency threshold: An allele is called for a sample at a + location if that fraction of the sample's aligned reads support it. This + matters most for diploid or aneuploid organisms, where heterozygous positions + are expected near 50%.

+ +

Minor allele frequency: Among the qualifying calls at a location, + the minor allele frequency is the percent that are not the major allele. Use 0 + to find every variant location within the group.

+ +

Percent samples with a base call: A location is only considered if + this fraction of your selected samples have a qualifying call there.

+ ]]> + + + + + + + Display the histogram of the values of this attribute + int + + + + + Display the histogram of the values of this attribute + int + + + + + + +``` + +- [ ] **Step 4: Verify both files parse** + +```bash +cd ~/workspaces/plasmodb/ApiCommonModel && for f in \ + Model/lib/wdk/model/questions/queries/variationQueries.xml \ + Model/lib/wdk/model/questions/variationQuestions.xml; do + python3 -c "import xml.etree.ElementTree as T,sys; T.parse('$f'); print('$f parses')" +done +``` + +Expected: both print `parses`. + +- [ ] **Step 5: Build and confirm the search is in the assembled model** + +```bash +cd ~/workspaces/agentic-veupath-dev && bin/veup-build.sh plasmodb wb model +``` + +Expected: completes with no `WdkModelException`. + +```bash +ssh cedar 'bash -lc "source /var/www/jbrestel.plasmodb.org/etc/setenv && wdkXml -model PlasmoDB"' \ + | grep -E "VariationsBy.VariationsByLocation:" +``` + +Expected: one line listing the query's params and columns — `organismSinglePick`, +`eda_sample_table_suffix`, `chromosomeOptionalForVariations`, `sequenceId`, `start_point`, +`end_point`, `variation_sample_meta`, `WebServicesPath`, `ReadFrequencyPercent`, +`MinPercentMinorAlleles`, `MinPercentIsolateCalls`, and columns `source_id, project_id, +PercentMinorAlleles, PercentIsolateCalls, Phenotype`. + +- [ ] **Step 6: Commit** + +```bash +cd ~/workspaces/plasmodb/ApiCommonModel +git add Model/lib/wdk/model/questions/queries/variationQueries.xml \ + Model/lib/wdk/model/questions/variationQuestions.xml +git commit -m "Add the VariationsByLocation search + +Reuses every param from VariationsByIsolateGroup and adds the region +restriction: chromosome (fallback), sequenceId (takes precedence), start and +end. All four are required by FindPolymorphismsWithSeqFilterPlugin. + +The chromosome fallback works through sequenceId's emptyValue='No Match', +which the plugin tests for by literal; documented in the query comment because +it reads as dead code from either side alone." +``` + +--- + +### Task 4: `VariationsByGeneIds` — query, question, build + +**Files:** +- Modify: `ApiCommonModel/Model/lib/wdk/model/questions/queries/variationQueries.xml` +- Modify: `ApiCommonModel/Model/lib/wdk/model/questions/variationQuestions.xml` + +- [ ] **Step 1: Confirm the plugin's gene-resolution SQL still works against this build** + +The plugin resolves your gene list to genomic intervals itself. Check the table it uses, and +record the interval you will assert against in Task 6: + +```bash +psql -h localhost -p 5432 -d unidb_shu_a -c " +SELECT source_id, sequence_id, start_min, end_max +FROM webready.GeneAttributes_p WHERE source_id = 'PF3D7_1133400'" +``` + +Expected: `PF3D7_1133400 | Pf3D7_11_v3 | 1292966 | 1296696`. If `webready.geneattributes_p` +did not exist, this search could not work at all and the plan would need revisiting. + +- [ ] **Step 2: Add the process query** + +In `variationQueries.xml`, inside the `VariationsBy` querySet, after `VariationsByLocation`: + +```xml + + + + + + + + The Organism defines the species identity of the samples and the genome + against which each sample's variants were called. + + + + + + + + + + + + + + + + + + +``` + +- [ ] **Step 3: Add the question** + +In `variationQuestions.xml`, after `VariationsByLocation`: + +```xml + + + + + + + + + + + + +
+ + Each sample's sequencing reads are aligned to the reference genome (Organism) + and variants are recorded for each sample based on the Read Frequency + Threshold. Then, scanning variant locations within your genes across the group + of samples, variants are returned if the Minor Allele Frequency and the Percent + samples with a base call are met. + +

Genes: Your gene IDs are resolved to each gene's genomic span, and + variants are returned by position within those spans. A variant in an intron or + UTR of one of your genes is therefore returned, since the span covers the whole + gene rather than only its coding sequence.

+ +

Organism: The Organism parameter defines the species of the samples + and the genome in which the variants are determined. Choose the organism your + genes belong to.

+ +

Samples: By default the group includes all samples from the Organism + you chose; you may narrow it using the sample characteristics. At least two + samples are required, since polymorphism within a group of one is undefined.

+ +

Read frequency threshold: An allele is called for a sample at a + location if that fraction of the sample's aligned reads support it. This + matters most for diploid or aneuploid organisms, where heterozygous positions + are expected near 50%.

+ +

Minor allele frequency: Among the qualifying calls at a location, the + minor allele frequency is the percent that are not the major allele. Use 0 to + find every variant location within the group.

+ +

Percent samples with a base call: A location is only considered if + this fraction of your selected samples have a qualifying call there.

+ ]]> + + + + + + + Display the histogram of the values of this attribute + int + + + + + Display the histogram of the values of this attribute + int + + + + + + +``` + +- [ ] **Step 4: Verify both files parse** + +```bash +cd ~/workspaces/plasmodb/ApiCommonModel && for f in \ + Model/lib/wdk/model/questions/queries/variationQueries.xml \ + Model/lib/wdk/model/questions/variationQuestions.xml; do + python3 -c "import xml.etree.ElementTree as T,sys; T.parse('$f'); print('$f parses')" +done +``` + +Expected: both print `parses`. + +- [ ] **Step 5: Build and confirm all four searches are in the assembled model** + +```bash +cd ~/workspaces/agentic-veupath-dev && bin/veup-build.sh plasmodb wb model +``` + +Expected: completes with no `WdkModelException`. + +```bash +ssh cedar 'bash -lc "source /var/www/jbrestel.plasmodb.org/etc/setenv && wdkXml -model PlasmoDB"' \ + | grep -oE "VariationsBy\.[A-Za-z]+:" | sort -u +``` + +Expected exactly four: `VariationsBy.VariationBySourceId:`, +`VariationsBy.VariationsByGeneIds:`, `VariationsBy.VariationsByIsolateGroup:`, +`VariationsBy.VariationsByLocation:`. + +- [ ] **Step 6: Commit** + +```bash +cd ~/workspaces/plasmodb/ApiCommonModel +git add Model/lib/wdk/model/questions/queries/variationQueries.xml \ + Model/lib/wdk/model/questions/variationQuestions.xml +git commit -m "Add the VariationsByGeneIds search + +No gene-to-variation join in the model: FindSnpsByGeneIdsPlugin resolves the +gene list to genomic intervals through webready.GeneAttributes_p and HSSS +filters variant positions by interval, so the relationship is positional and +apidb.VariationTranscriptProduct plays no part. + +Per-project gene defaults only for the three projects with variation data +loaded; the rest owe one when their data lands." +``` + +--- + +### Task 5: Category ontology rows + +**Files:** +- Modify: `ApiCommonModel/Model/lib/wdk/ontology/individuals.txt` + +The file is tab-delimited with load-bearing empty fields and a trailing tab. Derive each row by +substitution from the `VariationBySourceId` row; do not type them. + +- [ ] **Step 1: Look at the row you are copying** + +```bash +cd ~/workspaces/plasmodb/ApiCommonModel +grep -n "VariationQuestions.VariationBySourceId" Model/lib/wdk/ontology/individuals.txt | cat -A +``` + +Expected: one match showing `^I` between fields, `topic_0199` as parent, `search` as target +type, `menu` and `webservice` at the end, and a trailing `^I` before `$`. + +- [ ] **Step 2: Append both rows by substitution** + +```bash +cd ~/workspaces/plasmodb/ApiCommonModel +for s in VariationsByLocation VariationsByGeneIds; do + grep "VariationQuestions.VariationBySourceId" Model/lib/wdk/ontology/individuals.txt \ + | sed "s/VariationBySourceId/$s/g" >> Model/lib/wdk/ontology/individuals.txt +done +``` + +- [ ] **Step 3: Verify field counts match the source row** + +```bash +cd ~/workspaces/plasmodb/ApiCommonModel +awk -F'\t' '/VariationQuestions.Variation/ {print NF" fields "$6}' Model/lib/wdk/ontology/individuals.txt +``` + +Expected: four lines, all with the **same** field count, naming +`VariationQuestions.VariationBySourceId`, `...VariationsByIsolateGroup`, +`...VariationsByLocation`, `...VariationsByGeneIds`. A differing count means a shifted column +and a silently misfiled search — stop rather than patching by hand. + +- [ ] **Step 4: Build with `wb ontology`, NOT `wb model`** + +```bash +cd ~/workspaces/agentic-veupath-dev && bin/veup-build.sh plasmodb wb ontology +``` + +Expected: completes, with a line reporting `categories_merged.owl` saved. `wb model` here would +leave the OWL stale and both searches uncategorized, **with no error anywhere**. + +- [ ] **Step 5: Prove the OWL contains both, under the right parent** + +```bash +ssh cedar 'for s in VariationsByLocation VariationsByGeneIds; do echo -n "$s "; \ + grep -A3 "individuals.owl#VariationRecordClasses.VariationRecordClass.VariationQuestions.$s\"" \ + /var/www/jbrestel.plasmodb.org/gus_home/lib/wdk/ontology/categories_merged.owl \ + | grep -c "topic_0199"; done' +``` + +Expected: each prints `1` — the class exists and is `subClassOf topic_0199`. + +- [ ] **Step 6: Commit** + +```bash +cd ~/workspaces/plasmodb/ApiCommonModel +git add Model/lib/wdk/ontology/individuals.txt +git commit -m "Categorize VariationsByLocation and VariationsByGeneIds + +Same placement as the two existing variation searches: parent topic_0199, +targetType search, menu + webservice scopes. No searchCategory on either +question: it appears zero times in the assembled model, so menu grouping comes +from this file alone." +``` + +--- + +### Task 6: Browser verification + +Everything in Task 2 is verified by execution against the database. What only a live run can +establish is the two chains no test covers: the `emptyValue` to chromosome fallback, and gene +IDs through interval filtering to HSSS. + +**Files:** none. + +- [ ] **Step 1: Mark the logs** + +```bash +cd ~/workspaces/agentic-veupath-dev && bin/veup-logs.sh plasmodb mark loc-gene +``` + +- [ ] **Step 2: Open the app and confirm which site you are on** + +Load `https://jbrestel.plasmodb.org/a/app` in Chrome. It **redirects** to the webapp context. +Then, before trusting anything: + +```javascript +({origin: window.location.origin, base: window.location.pathname.split('/')[1]}) +``` + +Expected: `origin` is `https://jbrestel.plasmodb.org` and `base` is `plasmo.jbrestel`. If +origin reads `https://veupathdb.org`, **stop** — the tab bounced to autologin and every +relative fetch from here answers for production. Build all paths below from the `base` you +actually got, not from `/a/`. + +Note: `computer:screenshot` fails on this instance with `Script injection timed out`, including +on known-good pages. Use `javascript_tool` and the service endpoints; do not spend attempts on +screenshots. + +- [ ] **Step 3: Confirm both searches are registered** + +```javascript +const b = window.location.pathname.split('/')[1]; +fetch(`/${b}/service/record-types/variation`).then(r=>r.json()) + .then(d=>d.searches.map(s=>s.fullName).filter(n=>n.startsWith('VariationQuestions'))) +``` + +Expected: all four — `VariationBySourceId`, `VariationsByIsolateGroup`, `VariationsByLocation`, +`VariationsByGeneIds`. `/record-types` is project-filtered, so this is the source of truth. + +- [ ] **Step 4: `VariationsByLocation` — the chromosome vocabulary** + +```javascript +const b = window.location.pathname.split('/')[1]; +const q = await fetch(`/${b}/service/record-types/variation/searches/VariationsByLocation?expandParams=true`).then(r=>r.json()); +const chr = q.searchData.parameters.find(p=>p.name==='chromosomeOptionalForVariations'); +JSON.stringify({count: chr.vocabulary.length, terms: chr.vocabulary.map(v=>v[0])}) +``` + +Expected: 15 entries — `Choose chromosome` plus `01` through `14`. + +- [ ] **Step 5: `VariationsByLocation` — run it with the sequence box EMPTY** + +This is the step that exercises the `emptyValue` to `No Match` to chromosome-fallback chain. +Submit with `chromosomeOptionalForVariations` = the internal for chromosome 01 +(`Pf3D7_01_v3`), `sequenceId` left empty, `start_point` `0`, `end_point` `0`, all samples, +thresholds at 80% / 0 / 20. + +Expected: rows returned, and **every** returned ID begins `Variant_Pf3D7_01_v3_`. Check the +first page of IDs explicitly — a mixture of sequences would mean the region filter was ignored. + +| symptom | cause | +|---|---| +| zero results | the fallback did not fire; check `sequenceId`'s `emptyValue` reached the plugin | +| IDs from other chromosomes | the sequence argument never reached the bash script | +| missing required parameter | Task 1's rename is not installed — re-run Task 1 Steps 5 and 6 | + +- [ ] **Step 6: `VariationsByLocation` — run it with an explicit sequence and window** + +Submit `sequenceId` = `Pf3D7_11_v3`, `start_point` `1292966`, `end_point` `1296696`. + +Expected: fewer rows than Step 5, all with IDs of the form `Variant_Pf3D7_11_v3_` where +`1292966 <= n <= 1296696`. Extract `n` from the IDs and check the min and max against the +bounds rather than eyeballing. + +- [ ] **Step 7: `VariationsByGeneIds` — the first real exercise of `hsssGenomicLocationsFilter`** + +Submit with the PlasmoDB default gene `PF3D7_1133400`, all samples, thresholds as above. + +Expected: a non-empty result set, every ID of the form `Variant_Pf3D7_11_v3_` with +`1292966 <= n <= 1296696` — the same window as Step 6, because that is this gene's span. + +**A wrong ID form here surfaces as zero results with no error**, which is exactly why a +non-empty set is the assertion. The installed `hsssGenomicLocationsFilter` was confirmed to +carry the underscore ID join; if this returns nothing, re-check it with the pattern +single-quoted on the remote side: + +```bash +ssh cedar "grep -cF '\${contigSourceId}_\${location}' /var/www/jbrestel.plasmodb.org/gus_home/bin/hsssGenomicLocationsFilter" +``` + +Expected: `2`. (Without the escaping, the remote shell expands the variables and you count `_`.) + +- [ ] **Step 8: Confirm an ID resolves to a record page** + +Navigate to `//app/record/variation/` and confirm it renders +with its Genomic Location / Genetic variation / DNA polymorphism sections and no error. + +- [ ] **Step 9: Confirm both searches are in the category tree** + +```javascript +const b = window.location.pathname.split('/')[1]; +const c = await fetch(`/${b}/service/ontologies/Categories`).then(r=>r.json()); +const out=[]; +(function walk(n,parent){const p=n.properties||{};const nm=(p.name||[])[0]; + if(nm && nm.startsWith('VariationQuestions')) out.push({name:nm, parent}); + (n.children||[]).forEach(ch=>walk(ch,(p['EuPathDB alternative term']||p.label||[])[0]||parent)); +})(c.tree,'ROOT'); +JSON.stringify(out,null,1) +``` + +Expected: all four searches, each with `parent` `"Genetic variation"`. + +- [ ] **Step 10: Read the logs** + +```bash +cd ~/workspaces/agentic-veupath-dev && bin/veup-logs.sh plasmodb since loc-gene --quiet +``` + +Expected: the error logs report `silent:`. Also confirm the plugin used the paths you expect: + +```bash +cd ~/workspaces/agentic-veupath-dev && bin/veup-logs.sh plasmodb since loc-gene 2>&1 \ + | grep -oE "dnaseq/readFreq[0-9]+|hsssGenerate[A-Za-z]+" | sort | uniq -c +``` + +Expected: `hsssGeneratePolymorphismScript` for the two by-location runs and +`hsssGenerateGenomicLocationsScript` for the by-gene run. + +**Any ERROR lines you cause yourself by hand-rolling malformed service requests must be +reported as such, not as "the logs were silent."** That happened on the previous search. + +- [ ] **Step 11: Report** + +Report: the four registered searches; the chromosome vocabulary count; for each of the three +runs, the row count and the observed min/max location against the expected window; one ID that +resolved to a record page; the category-tree parents; the log verdict, distinguishing +self-inflicted errors from real ones. If a step failed, give the symptom and the diagnosis from +the tables above rather than guessing at a fix. + +--- + +## Out of scope + +- **`VariationsByTwoIsolateGroups`** — its own spec and plan. Different plugin + (`FindMajorAllelesPlugin`), 11 `wsColumn`s, five doubled threshold params, and **two new + EDA-driven filter params**, because `sharedParams.ngsSnp_strain_meta_a`/`_m` sit inside a + commented-out region and are not in the model. That spec must not forget the plugin's + hardcoded `_a`/`_m` names. +- **`NgsSnpsByTwoIsolateGroupsWiz`** — a sixth search using the `*_wiz` params; decide whether + to port it at all alongside `ByTwoIsolateGroups`. +- **Per-gene coding consequences** from `apidb.VariationTranscriptProduct`. A feature, not a port. +- **Deleting the dead `snpParams.xml`** and the commented-out snp regions of `sharedParams.xml`. +- **`ReadFrequencyPercent` as a functional parameter.** On haploid organisms all four + `readFreq*` directories hold identical data, because the upstream caller runs + `freebayes --min-alternate-fraction 0.8`. The param selects its directory correctly. Do not + attempt to verify that changing it changes results; it will not, and that is not a bug in + these searches. Design §9. From 87b5db50b9ff3f8f24fba60d5dac94fbde472554 Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Wed, 5 Aug 2026 16:03:46 -0400 Subject: [PATCH 47/98] Fix the plan's installed-jar check: read the class, do not grep the lib dir Jar entries are compressed, so grep -rl over WEB-INF/lib matches nothing whether or not the string is present -- a check that can only fail. Now unzips the class and asserts new=1 old=0, the old string's absence being the stronger signal. The Task 1 implementer hit this. --- ...-05-variations-by-location-and-gene-ids.md | 26 ++++++++++--------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/docs/superpowers/plans/2026-08-05-variations-by-location-and-gene-ids.md b/docs/superpowers/plans/2026-08-05-variations-by-location-and-gene-ids.md index 33939c35a4..70a3bfbb9d 100644 --- a/docs/superpowers/plans/2026-08-05-variations-by-location-and-gene-ids.md +++ b/docs/superpowers/plans/2026-08-05-variations-by-location-and-gene-ids.md @@ -135,22 +135,24 @@ cd ~/workspaces/agentic-veupath-dev && bin/veup-build.sh plasmodb reload Expected: `OK - Reloaded application at context path [/plasmo.jbrestel]`. -```bash -ssh cedar "bash -lc 'cd /var/www/PlasmoDB/plasmo.jbrestel/webapp/WEB-INF/lib && \ - for j in *.jar; do unzip -p \$j org/apidb/apicomplexa/wsfplugin/highspeedsnpsearch/FindPolymorphismsWithSeqFilterPlugin.class 2>/dev/null | \ - strings | grep -l chromosomeOptionalForVariations >/dev/null && echo \$j; done'" 2>/dev/null -``` - -Expected: at least one jar name printed. If nothing prints, try the simpler form below; the -point is to confirm the *installed* class carries the new string, not just the source tree. +Jar entries are compressed, so **`grep` over the lib directory finds nothing whether or not the +string is there** — a check that can only fail. Read the class out of the jar instead, and test +for both strings; the old one being *gone* is the stronger signal: ```bash -ssh cedar "bash -lc 'grep -rl chromosomeOptionalForVariations /var/www/PlasmoDB/plasmo.jbrestel/webapp/WEB-INF/lib/ 2>/dev/null | head'" +ssh cedar "bash -lc 'J=/var/www/PlasmoDB/plasmo.jbrestel/webapp/WEB-INF/lib/api-common-websvc-wsfplugin-1.0.0.jar; \ + C=org/apidb/apicomplexa/wsfplugin/highspeedsnpsearch/FindPolymorphismsWithSeqFilterPlugin.class; \ + echo -n \"new=\"; unzip -p \$J \$C | strings | grep -c chromosomeOptionalForVariations; \ + echo -n \"old=\"; unzip -p \$J \$C | strings | grep -c chromosomeOptionalForNgsSnps'" ``` -Expected: one or more jar paths. **A pass here is what makes Task 6 meaningful** — with the old -string installed, the search fails at run time with a missing-required-parameter error and the -model looks wrong when it is not. +Expected: `new=1` and `old=0`. Also check the jar's mtime matches the build you just ran +(`ls -l $J`). If the jar name has changed, find it with +`ssh cedar 'ls /var/www/PlasmoDB/plasmo.jbrestel/webapp/WEB-INF/lib/ | grep wsfplugin'`. + +**A pass here is what makes Task 6 meaningful** — with the old string installed, the search +fails at run time with a missing-required-parameter error and the model looks wrong when it is +not. --- From 6b389027552e4b9b3ace90ad5bce809ba5c5fd9c Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Wed, 5 Aug 2026 16:05:26 -0400 Subject: [PATCH 48/98] Add chromosomeOptionalForVariations param and its vocabulary Rewritten, not reused. The snp original's vocabulary reaches organismVQ.withNgsSNPs, which reads the nonexistent apidbtuning.snpstrains, and keys on org_abbrev while our organism param carries the taxon name (0 rows if you try it). Keyed on organism instead: 15 rows in 26ms. internal is the sequence source_id (chromosome 01 gives Pf3D7_01_v3), which is the identifier the HSSS files use, so the value feeds the position filter with no mapping layer. Co-Authored-By: Claude Opus 5 --- .../questions/params/variationParams.xml | 61 +++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/Model/lib/wdk/model/questions/params/variationParams.xml b/Model/lib/wdk/model/questions/params/variationParams.xml index 75de9dc601..d119272d45 100644 --- a/Model/lib/wdk/model/questions/params/variationParams.xml +++ b/Model/lib/wdk/model/questions/params/variationParams.xml @@ -169,6 +169,33 @@ \d\d?|100 + + + + + + + + + + + + + + + + + + + From ef12c771d91e6820709ba220916f410ab4ef1c40 Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Wed, 5 Aug 2026 16:09:38 -0400 Subject: [PATCH 49/98] Add the VariationsByLocation search Reuses every param from VariationsByIsolateGroup and adds the region restriction: chromosome (fallback), sequenceId (takes precedence), start and end. All four are required by FindPolymorphismsWithSeqFilterPlugin. The chromosome fallback works through sequenceId's emptyValue='No Match', which the plugin tests for by literal; documented in the query comment because it reads as dead code from either side alone. --- .../questions/queries/variationQueries.xml | 41 +++++++++ .../model/questions/variationQuestions.xml | 84 +++++++++++++++++++ 2 files changed, 125 insertions(+) diff --git a/Model/lib/wdk/model/questions/queries/variationQueries.xml b/Model/lib/wdk/model/questions/queries/variationQueries.xml index 4fffc556eb..a689b694e2 100644 --- a/Model/lib/wdk/model/questions/queries/variationQueries.xml +++ b/Model/lib/wdk/model/questions/queries/variationQueries.xml @@ -68,6 +68,47 @@ + + + + + + + + The Organism defines the species identity of the samples and the genome + against which each sample's variants were called. + + + + + + + + + + + + + + + + + + + + diff --git a/Model/lib/wdk/model/questions/variationQuestions.xml b/Model/lib/wdk/model/questions/variationQuestions.xml index 9e7e6c2ba1..7a88a5f5ca 100644 --- a/Model/lib/wdk/model/questions/variationQuestions.xml +++ b/Model/lib/wdk/model/questions/variationQuestions.xml @@ -129,6 +129,90 @@ + + + + + + + + + + + + +
+ + Each sample's sequencing reads are aligned to the reference genome (Organism) + and variants are recorded for each sample based on the Read Frequency + Threshold. Then, scanning variant locations within your region across the + group of samples, variants are returned if the Minor Allele Frequency and the + Percent samples with a base call are met. + +

Defining the region: Either choose a Chromosome, or enter a Genomic + sequence ID. A sequence ID you enter takes precedence; the Chromosome menu is + used when you leave the sequence box empty. Start and End restrict the region + further, and an End of 0 means "to the end of the sequence".

+ +

Organism: The Organism parameter defines the species of the samples + and the genome in which the variants are determined. Choosing an Organism + focuses the Samples parameter to the samples of that organism.

+ +

Samples: By default the group includes all samples from the Organism + you chose; you may narrow it using the sample characteristics. At least two + samples are required, since polymorphism within a group of one is undefined.

+ +

Read frequency threshold: An allele is called for a sample at a + location if that fraction of the sample's aligned reads support it. This + matters most for diploid or aneuploid organisms, where heterozygous positions + are expected near 50%.

+ +

Minor allele frequency: Among the qualifying calls at a location, + the minor allele frequency is the percent that are not the major allele. Use 0 + to find every variant location within the group.

+ +

Percent samples with a base call: A location is only considered if + this fraction of your selected samples have a qualifying call there.

+ ]]> + + + + + + + Display the histogram of the values of this attribute + int + + + + + Display the histogram of the values of this attribute + int + + + + + + + From 45a2e2a0b5f15b3b06e0b3c1e72cad885017963d Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Wed, 5 Aug 2026 16:14:24 -0400 Subject: [PATCH 50/98] Add the VariationsByGeneIds search No gene-to-variation join in the model: FindSnpsByGeneIdsPlugin resolves the gene list to genomic intervals through webready.GeneAttributes_p and HSSS filters variant positions by interval, so the relationship is positional and apidb.VariationTranscriptProduct plays no part. Per-project gene defaults only for the three projects with variation data loaded; the rest owe one when their data lands. Co-Authored-By: Claude Opus 5 --- .../questions/queries/variationQueries.xml | 40 +++++++++ .../model/questions/variationQuestions.xml | 82 +++++++++++++++++++ 2 files changed, 122 insertions(+) diff --git a/Model/lib/wdk/model/questions/queries/variationQueries.xml b/Model/lib/wdk/model/questions/queries/variationQueries.xml index a689b694e2..69610b9a67 100644 --- a/Model/lib/wdk/model/questions/queries/variationQueries.xml +++ b/Model/lib/wdk/model/questions/queries/variationQueries.xml @@ -109,6 +109,46 @@ + + + + + + + + The Organism defines the species identity of the samples and the genome + against which each sample's variants were called. + + + + + + + + + + + + + + + + + + + diff --git a/Model/lib/wdk/model/questions/variationQuestions.xml b/Model/lib/wdk/model/questions/variationQuestions.xml index 7a88a5f5ca..c0f83f9f92 100644 --- a/Model/lib/wdk/model/questions/variationQuestions.xml +++ b/Model/lib/wdk/model/questions/variationQuestions.xml @@ -213,6 +213,88 @@ + + + + + + + + + + + + +
+ + Each sample's sequencing reads are aligned to the reference genome (Organism) + and variants are recorded for each sample based on the Read Frequency + Threshold. Then, scanning variant locations within your genes across the group + of samples, variants are returned if the Minor Allele Frequency and the Percent + samples with a base call are met. + +

Genes: Your gene IDs are resolved to each gene's genomic span, and + variants are returned by position within those spans. A variant in an intron or + UTR of one of your genes is therefore returned, since the span covers the whole + gene rather than only its coding sequence.

+ +

Organism: The Organism parameter defines the species of the samples + and the genome in which the variants are determined. Choose the organism your + genes belong to.

+ +

Samples: By default the group includes all samples from the Organism + you chose; you may narrow it using the sample characteristics. At least two + samples are required, since polymorphism within a group of one is undefined.

+ +

Read frequency threshold: An allele is called for a sample at a + location if that fraction of the sample's aligned reads support it. This + matters most for diploid or aneuploid organisms, where heterozygous positions + are expected near 50%.

+ +

Minor allele frequency: Among the qualifying calls at a location, the + minor allele frequency is the percent that are not the major allele. Use 0 to + find every variant location within the group.

+ +

Percent samples with a base call: A location is only considered if + this fraction of your selected samples have a qualifying call there.

+ ]]> + + + + + + + Display the histogram of the values of this attribute + int + + + + + Display the histogram of the values of this attribute + int + + + + + + + From fb976c0b310b96d2ef8a91b57162f81d65815fc4 Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Wed, 5 Aug 2026 16:17:59 -0400 Subject: [PATCH 51/98] Categorize VariationsByLocation and VariationsByGeneIds Same placement as the two existing variation searches: parent topic_0199, targetType search, menu + webservice scopes. No searchCategory on either question: it appears zero times in the assembled model, so menu grouping comes from this file alone. --- Model/lib/wdk/ontology/individuals.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Model/lib/wdk/ontology/individuals.txt b/Model/lib/wdk/ontology/individuals.txt index 143c0964f6..9873ea59dd 100644 --- a/Model/lib/wdk/ontology/individuals.txt +++ b/Model/lib/wdk/ontology/individuals.txt @@ -1157,3 +1157,5 @@ VariationRecordClasses.VariationRecordClass.TranscriptProducts http://edamontolo VariationRecordClasses.VariationRecordClass.PredictedEffects http://edamontology.org/topic_0199 Genetic Variation VariationRecordClasses.VariationRecordClass table PredictedEffects record download VariationRecordClasses.VariationRecordClass.VariationQuestions.VariationBySourceId http://edamontology.org/topic_0199 Genetic Variation VariationRecordClasses.VariationRecordClass search VariationQuestions.VariationBySourceId menu webservice VariationRecordClasses.VariationRecordClass.VariationQuestions.VariationsByIsolateGroup http://edamontology.org/topic_0199 Genetic Variation VariationRecordClasses.VariationRecordClass search VariationQuestions.VariationsByIsolateGroup menu webservice +VariationRecordClasses.VariationRecordClass.VariationQuestions.VariationsByLocation http://edamontology.org/topic_0199 Genetic Variation VariationRecordClasses.VariationRecordClass search VariationQuestions.VariationsByLocation menu webservice +VariationRecordClasses.VariationRecordClass.VariationQuestions.VariationsByGeneIds http://edamontology.org/topic_0199 Genetic Variation VariationRecordClasses.VariationRecordClass search VariationQuestions.VariationsByGeneIds menu webservice From a74be0fdf8f5715a22e4827d4960bb3a3e29ddfe Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Wed, 5 Aug 2026 16:34:55 -0400 Subject: [PATCH 52/98] Design VariationsByTwoIsolateGroups, the last of the five snp ports Not 'the last one plus a param'. FindMajorAllelesPlugin extends HighSpeedSnpSearchAbstractPlugin directly rather than FindPolymorphismsPlugin, so it inherits no strain-filter accessor and declares all ten required params itself: 12 result columns, an 11-column results file, and two sample filters. The two filters are symmetric _a/_b copies of variation_sample_meta sharing its existing EDA queries -- WDK clones a dependent param's queries per param, which is how the snp original had two filters on one query. No new SQL in this spec. Deliberately no minSelectedCount, unlike the one-group filter: comparing major alleles between two groups of one is meaningful, unlike polymorphism within one. Set A reuses the unsuffixed ReadFrequencyPercent and MinPercentIsolateCalls because those are the plugin's own Set A constants; only Set B gets ...Two variants. Four new threshold params. Records uniq-value-params as the piece most likely to be dropped: it forbids A = B, is used four more times in geneQuestions, and its absence cannot be detected by any build or service check -- only by noticing the form accepts A = B. --- ...variations-by-two-isolate-groups-design.md | 308 ++++++++++++++++++ 1 file changed, 308 insertions(+) create mode 100644 docs/superpowers/specs/2026-08-05-variations-by-two-isolate-groups-design.md diff --git a/docs/superpowers/specs/2026-08-05-variations-by-two-isolate-groups-design.md b/docs/superpowers/specs/2026-08-05-variations-by-two-isolate-groups-design.md new file mode 100644 index 0000000000..a12d8c4129 --- /dev/null +++ b/docs/superpowers/specs/2026-08-05-variations-by-two-isolate-groups-design.md @@ -0,0 +1,308 @@ +# `VariationsByTwoIsolateGroups` — design + +**Date:** 2026-08-05 +**Status:** approved +**Scope:** The last of the five `snp` searches to port — `NgsSnpsByTwoIsolateGroups` becomes +`VariationsByTwoIsolateGroups`. Finds loci whose *major allele* differs between two +user-chosen groups of samples. +**Implementation target:** `ApiCommonModel`, plus two constants in `ApiCommonWebService`. +Branch `dnaseq-merge-experiments`. +**Prerequisites, all implemented and verified live:** +- `2026-08-05-hsss-variation-plumbing-design.md` (`ApiCommonWebService`) +- `2026-08-05-variations-by-isolate-group-design.md` — supplies the filter this one doubles +- `2026-08-05-variations-by-location-and-gene-ids-design.md` + +Section references: `group §` → the by-isolate-group spec, `locgene §` → the location/genes spec. + +## 1. Purpose, and why this one is not like the other three + +The three ported HSSS searches so far all answer "what varies *within* one group of samples," +optionally restricted to a region. This one asks a different question: **take two groups, find +the major allele in each, and return the loci where those major alleles disagree.** It is the +search you use to find markers that distinguish two populations. + +That difference in question produces a difference in shape, and this is the one port where +"same as the last one plus a param" does not hold: + +| | one group | two groups | +|---|---|---| +| plugin | `FindPolymorphismsPlugin` | `FindMajorAllelesPlugin` | +| plugin's parent | `FindPolymorphismsPlugin` chain | **`HighSpeedSnpSearchAbstractPlugin` directly** | +| threshold of interest | *minor* allele frequency | *major* allele frequency, per group | +| result columns | 5 | **12** | +| results-file columns | 4 | **11** | +| sample filters | 1 | **2, which must differ** | + +The middle row matters: because `FindMajorAllelesPlugin` extends the abstract plugin rather +than `FindPolymorphismsPlugin`, it inherits **no** `getStrainFilterParamName()`. Its filter +param names are its own constants, which is why §4 renames two of them rather than one. + +## 2. Reused unchanged + +Three of the ten required params are already ours, under exactly the names the plugin wants: + +| Component | Where | +|---|---| +| `organismVQ.withVariationsTree` | `organismParams.xml`, group §4.1 | +| `variationParams.eda_sample_table_suffix` + `VariationVQ.EdaSampleTableSuffix` | group §4.2 | +| `VariationVQ.SamplesMetadataByStudy` / `SampleOntologyByStudy` | group §5 — **shared by all three filter params**, see §4.2 | +| `variationParams.WebServicesPath` | group §4.4 | +| `variationParams.ReadFrequencyPercent` | group §4.4 — serves **Set A** | +| `variationParams.MinPercentIsolateCalls` | group §4.4 — serves **Set A** | + +Note the asymmetry in the last two: the plugin's Set A constants are the *unsuffixed* names the +one-group search already defines, and only Set B gets `...Two` variants. That is the plugin's +choice, not ours, and it means Set A's read-frequency and percent-called params are literally +the same param objects as the one-group search's, distinguished only by a `prompt` override on +the `paramRef` (as the snp original did). + +## 3. The plugin contract (read this before writing any XML) + +`FindMajorAllelesPlugin.getRequiredParameterNames()` returns exactly ten names. Miss one and +the search fails at run time as a missing required parameter, not at build time. + +| Plugin constant | Required param name | Status | +|---|---|---| +| `PARAM_ORGANISM` | `organismSinglePick` | exists | +| `PARAM_WEBSVCPATH` | `WebServicesPath` | exists | +| `PARAM_STRAIN_FILTER_A` | `ngsSnp_strain_meta_a` → **`variation_sample_meta_a`** | §4, rename + new | +| `PARAM_READ_FREQ_PERCENT_A` | `ReadFrequencyPercent` | exists | +| `PARAM_MIN_PERCENT_KNOWNS_A` | `MinPercentIsolateCalls` | exists | +| `PARAM_MIN_PERCENT_MAJOR_ALLELES_A` | `MinPercentMajorAlleles` | **new** (§5) | +| `PARAM_STRAIN_FILTER_B` | `ngsSnp_strain_meta_m` → **`variation_sample_meta_b`** | §4, rename + new | +| `PARAM_READ_FREQ_PERCENT_B` | `ReadFrequencyPercentTwo` | **new** (§5) | +| `PARAM_MIN_PERCENT_KNOWNS_B` | `MinPercentIsolateCallsTwo` | **new** (§5) | +| `PARAM_MIN_PERCENT_MAJOR_ALLELES_B` | `MinPercentMajorAllelesTwo` | **new** (§5) | + +`getColumns()` returns twelve, and `makeResultRow` throws unless the results file has **exactly +11** tab-separated columns (`project_id` is supplied by the plugin, not the file): + +``` +SourceId, ProjectId, +MajorAlleleA, MajorAllelePctA, IsTriallelicA, MajorProductA, MajorProductIsVariableA, +MajorAlleleB, MajorAllelePctB, IsTriallelicB, MajorProductB, MajorProductIsVariableB +``` + +Two more inherited facts worth knowing: `getReconstructCmdName()` returns +`hsssReconstructSnpId` — the script whose ID separator was fixed in the plumbing work, so IDs +arrive as `Variant__` — and the plugin passes `suffix = "NULL"` plus +`strains_are_names = 1`, so the filter's EDA sample stable IDs are used directly as strain +names, exactly as in the one-group search (group §4.3). + +Observation, recorded and **not** acted on: the Set A path null-checks its strains twice +(redundantly) while the Set B path never null-checks `strainsB` after resolving it. An empty +Set B would therefore reach `writeStrainsFile` unguarded. Since `uniq-value-params` (§6.3) and +the filter's own defaults make an empty Set B hard to produce through the UI, this is left +alone rather than fixed under cover of a port. + +## 4. The two sample filters + +### 4.1 Naming: symmetric `_a` / `_b` + +```java +- public static final String PARAM_STRAIN_FILTER_A = "ngsSnp_strain_meta_a"; +- public static final String PARAM_STRAIN_FILTER_B = "ngsSnp_strain_meta_m"; ++ public static final String PARAM_STRAIN_FILTER_A = "variation_sample_meta_a"; ++ public static final String PARAM_STRAIN_FILTER_B = "variation_sample_meta_b"; +``` + +Two decisions here. **The `_m` becomes `_b`** — it is the odd one out in the original, whose +prompts already read "Set B Isolates"; nothing in the plugin distinguishes it beyond being the +second group. And **Set A gets its own `_a` param** rather than reusing the unsuffixed +`variation_sample_meta`. Reuse was the cheaper option (one fewer definition, and Set A would +literally be the one-group search's filter) and was rejected: a search declaring an unsuffixed +A beside a suffixed B reads as though the two were different in kind. Symmetry is worth one +duplicated definition, and it makes `uniq-value-params` (§6.3) obvious at a glance. + +This continues the rename already applied twice — `variation_sample_meta` (plumbing §3.5) and +`chromosomeOptionalForVariations` (locgene §5). After this, no snp-era param name survives in +any variation search. + +### 4.2 Definitions: two new params, zero new queries + +```xml + +``` + +and the same again for `_b` with `prompt="Set B Samples"`. **Both reuse the existing EDA +queries** — no new SQL anywhere in this spec. WDK resolves a dependent param's queries by +`query.clone()` followed by `setContextParam(this)` (`AbstractDependentParam:230-231`, inside +`resolveDependentQuery`), so +three filter params sharing two query definitions get three independent instances. The snp +original did exactly this: `ngsSnp_strain_meta_a` and `_m` both pointed at +`SnpVQ.SamplesMetadataByOrganism`. + +This is what "constructed the same way as in the one group question" cashes out to: same +metadata query, same ontology query, same 27-node/7-category tree over 216 samples, same +`sample_stable_id` internals that HSSS accepts as strain names. + +**No `minSelectedCount`, deliberately** — and this is a real difference from +`variation_sample_meta`, which sets `minSelectedCount="2"`. Polymorphism *within* a group of +one is undefined, which is why the one-group search requires two. Comparing the major allele +*between* two groups of one is perfectly meaningful: it is a strain-versus-strain comparison. +The snp original set no minimum on either group. Do not add one. + +## 5. The four new threshold params + +Copied from `snpParams.xml` into `variationParams`, for the same reason as the previous four: +`snpParams` is imported inside a commented-out block and is absent from the assembled model, so +a `paramRef` to it fails model load (group §4.4). + +| param | shape | notes | +|---|---|---| +| `MinPercentMajorAlleles` | `stringParam`, `number="true"`, ``, `\d\d?|100` | prompt "Major allele frequency >= "; serves **Set A** | +| `MinPercentMajorAllelesTwo` | identical shape and default | serves **Set B** | +| `MinPercentIsolateCallsTwo` | `stringParam`, `number="true"`, ``, `\d\d?|100` | prompt "Min percent isolates with calls >= " | +| `ReadFrequencyPercentTwo` | `enumParam`, `quote="false"`, terms `80%`/`60%`/`40%`/`20%` → internals `80`/`60`/`40`/`20` | the four internals must match the `readFreq*` directories exactly | + +Three deliberate departures, consistent with the earlier ports: + +- **One `ReadFrequencyPercentTwo`, not two.** The original declares it twice + (`excludeProjects="ToxoDB"` / `includeProjects="ToxoDB"`), differing only in help text. +- **Help text says "samples", not "isolates"**, matching EDA vocabulary and the display names. + The scientific content is carried over intact, including the genuinely useful note that 100% + is permissible and is the *most stringent* setting for a major-allele threshold — a point + that reads as counterintuitive until you notice the search first identifies an allele in one + set and then compares it against the other. +- **`MinPercentIsolateCallsTwo` keeps its original prompt wording** ("Min percent isolates with + calls >= ") rather than being harmonised with `MinPercentIsolateCalls`' longer phrasing + ("Percent samples with a base call >= "). Flagged because it is an inconsistency a reader + will notice: the two prompts sit side by side in one form. Harmonising is a one-line change + and is left to the implementer's judgement at review time; the param **name** must not change. + +## 6. The search + +### 6.1 `VariationsBy.VariationsByTwoIsolateGroups` + +`processName="org.apidb.apicomplexa.wsfplugin.highspeedsnpsearch.FindMajorAllelesPlugin"`, +with params in the original's order — organism, then all of Set A, then all of Set B — so the +form reads as two parallel blocks: + +```xml + + + + + + + + + + + + + +``` + +The `prompt` overrides are load-bearing for usability, not decoration: without them Set A's +three thresholds carry the one-group search's generic labels and the form shows two +indistinguishable "Read frequency threshold" fields. + +Twelve `wsColumn`s per §3, `source_id` at width 60, `project_id` at 20, the two `*Pct*` +columns `columnType="float"` at width 8, the rest narrow. + +### 6.2 The question + +```xml + + +``` + +Thirteen summary columns is a lot, and it is the original's choice reproduced: the whole point +of the search is the side-by-side comparison, and dropping either set's columns would hide half +the answer. `displayName` says **Samples** rather than the original's "Isolates". + +Ten `dynamicAttributes`, all `sortable="false"` as in the original (the plugin emits them as +strings, so a lexical sort on `MajorAllelePctA` would mis-order), with `Set A ` / `Set B ` +display-name prefixes carried over verbatim. + +### 6.3 `uniq-value-params` — the piece most likely to be dropped + +```xml + + variation_sample_meta_a + variation_sample_meta_b + +``` + +This declares that the two params must not hold the same value; comparing a group against +itself returns nothing useful. It is **not** snp cruft — `geneQuestions.xml` uses the same +property four times for fold-change reference-versus-comparison sample pairs. + +Enforcement is client-side, in `web-monorepo`, which is not checked out in this workspace — so +the convention is verifiable but the implementation is not readable from here. Consequence for +verification: **its absence cannot be detected by any build or service check**, only by +noticing in the browser that the form lets you submit A = B. Port it, and confirm it in the +rendered form. + +## 7. Ontology + +One row in `Model/lib/wdk/ontology/individuals.txt`, parent +`http://edamontology.org/topic_0199`, `targetType` `search`, scopes `menu` + `webservice` — +identical placement to the four existing variation searches, derived by substitution from the +`VariationBySourceId` row. Build with **`wb ontology`**, not `wb model`. + +No `searchCategory`: it appears zero times in the assembled model (locgene §7). + +## 8. Verification + +The Java rename must be built and installed (`bld ApiCommonWebService`) and the webapp reloaded +before any run, or the search fails as a missing required parameter while the model looks wrong. +Confirm the installed jar carries both new strings and neither old one. + +Then, in the browser (screenshots are unavailable on this instance; use `javascript_tool` and +the service, and check `window.location.origin` first): + +1. `/service/record-types/variation` lists all **five** variation searches. +2. The form renders two parallel Set A / Set B blocks with distinct prompts, and both sample + filters populate with 216 samples under the 7-category tree. +3. **The disjoint-groups run.** `country` splits the Pf samples cleanly — 147 French Guiana, 69 + Senegal (verified during the one-group work). Set A = Senegal, Set B = French Guiana, major + allele thresholds at the 80 default. Expect a non-empty result set, and for each row + `MajorAlleleA != MajorAlleleB` — that inequality *is* the search's definition, so a row + violating it means the comparison is broken. +4. **The symmetry check.** Swap the two groups. The total count must be identical, because + "major alleles disagree" is symmetric. A differing count means Set B's thresholds are not + being applied the way Set A's are, which is exactly the sort of copy-paste asymmetry the + doubled params invite. +5. **The `uniq-value-params` check.** Set A = Set B = Senegal. The form should refuse it. If it + submits, the property did not take effect (§6.3). +6. Returned IDs are `Variant__` and resolve to record pages. +7. `veup-logs.sh plasmodb mark/since` around all of it: error logs silent, and any errors caused + by hand-rolled service requests reported as self-inflicted rather than counted as clean. + +Note for step 3: a `datasetParam`-free search, so no dataset creation is needed — but the two +filter values are `filterParam` JSON, and the two must differ or step 5's guard is what you are +testing instead. + +## 9. Out of scope + +- **`NgsSnpsByTwoIsolateGroupsWiz`** — a sixth search (PlasmoDB/UniDB only) driven by the + `*_wiz` params, which live in the same commented-out region of `sharedParams.xml` as + `ngsSnp_strain_meta_a`/`_m`. Whether the wizard flow is still wanted is a product question, + not a porting one. **After this spec ships, the five-search port is complete** and this is + the only snp-era search left unported. +- **Deleting the dead snp XML.** With this spec, every param the five searches need lives in + `variationParams`. `snpParams.xml` and the commented-out snp regions of `sharedParams.xml` + then have no remaining consumer — but removal has its own blast radius (`recordParams.xml`, + `spanQuestions.xml`, `SnpsBySpanLogic`) and is a separate change. +- **Fixing the Set B null-check asymmetry** in `FindMajorAllelesPlugin` (§3). +- **Harmonising the two percent-called prompts** (§5) beyond the implementer's call. +- **Per-gene coding consequences** from `apidb.VariationTranscriptProduct`. +- **`ReadFrequencyPercent` as a functional parameter.** On haploid organisms all four + `readFreq*` directories hold identical data, because the upstream caller runs + `freebayes --min-alternate-fraction 0.8`. Both read-frequency params here select their + directory correctly; changing either will not change results on a haploid site, and that is + not a defect in this search. This bites twice as hard here, since the symmetry check in §8.4 + would also pass trivially if the two sets read the same files — so treat §8.4 as testing the + *threshold* plumbing, not the read-frequency plumbing. From 46562253278c9985a9d5846e4d2bcfaa515eda53 Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Wed, 5 Aug 2026 16:40:20 -0400 Subject: [PATCH 53/98] Add implementation plan for VariationsByTwoIsolateGroups Five tasks: rename both filter constants in ApiCommonWebService (with an installed-jar check asserting both new strings present and both old absent), the six new params, the query and question, the ontology row, then browser verification. Verification asserts the search's definition rather than a row count: every returned row must have MajorAlleleA != MajorAlleleB, and swapping the two groups must give an identical total since the comparison is symmetric. The uniq-value-params guard is browser-only by nature and gets its own step. Self-reviewed: every XML block parses, all param and query references resolve, and the 12 wsColumns / 10 dynamicAttributes / 13 summary columns agree with each other and with the plugin's contract. --- ...-08-05-variations-by-two-isolate-groups.md | 809 ++++++++++++++++++ 1 file changed, 809 insertions(+) create mode 100644 docs/superpowers/plans/2026-08-05-variations-by-two-isolate-groups.md diff --git a/docs/superpowers/plans/2026-08-05-variations-by-two-isolate-groups.md b/docs/superpowers/plans/2026-08-05-variations-by-two-isolate-groups.md new file mode 100644 index 0000000000..befba938fc --- /dev/null +++ b/docs/superpowers/plans/2026-08-05-variations-by-two-isolate-groups.md @@ -0,0 +1,809 @@ +# `VariationsByTwoIsolateGroups` Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Add the fifth and last ported snp search — find loci whose major allele differs between two user-chosen groups of samples — completing the `snp` to `variation` migration. + +**Architecture:** Two constants renamed in `ApiCommonWebService`; in `ApiCommonModel`, two new filter params (sharing the existing EDA queries — **no new SQL**), four new threshold params, one `processQuery`, one question, one ontology row. The design is `docs/superpowers/specs/2026-08-05-variations-by-two-isolate-groups-design.md` — read it; this plan implements it and does not restate its reasoning. + +**Tech Stack:** WDK model XML; Java (two string constants); the `agentic-veupath-dev` control plane for remote builds on `cedar`; Claude in Chrome for verification. + +--- + +## Orientation + +**Repos**, both on branch **`dnaseq-merge-experiments`**, never `main`: + +| | | +|---|---| +| `~/workspaces/plasmodb/ApiCommonWebService` | Task 1 only | +| `~/workspaces/plasmodb/ApiCommonModel` | Tasks 2–4 | +| `~/workspaces/agentic-veupath-dev` | control plane — run `bin/veup-*.sh` from **here** | + +Local edits reach `cedar` through a running `mutagen` sync. Builds run remotely. + +**There is no unit-test framework for WDK model XML.** Verification is a structural XML check, a remote build (proves references resolve), `wdkXml` (proves presence in the *assembled* model), and the browser. + +**Six traps, every one of which has already cost time on this feature:** + +1. **Flags go BEFORE the profile name.** `bin/veup-build.sh plasmodb wb model --dry-run` silently drops the flag **and runs for real**. Commands below are complete; add nothing. +2. **XML forbids `--` inside ``.** All comments below are checked; keep double hyphens out if you reword. `` is exempt. +3. **`wdkXml` prints attributes single-quoted** (`name='x'`). A double-quoted grep pattern matches nothing regardless of model content. +4. **`dynamicAttributes` is mandatory** for any `wsColumn` named in `attributesList`. Ten of them here. Omitting the block fails the build with `Summary attribute field [...] is invalid`. +5. **Jar entries are compressed** — `grep` over `WEB-INF/lib` finds nothing whether or not a string is present. Unzip the class (Task 1 Step 6). +6. **A remote grep for a `$`-containing pattern** gets expanded by the remote shell unless single-quoted on the remote side: `ssh host "... '\$foo' ..."`. + +**This search's own trap:** `uniq-value-params` (Task 3 Step 3) forbids Set A = Set B. Its absence **cannot be detected by any build or service check** — only by noticing in the browser that the form accepts A = B. Do not drop it, and do verify it in Task 5. + +--- + +## File Structure + +| File | Change | Responsibility | +|---|---|---| +| `ApiCommonWebService/.../FindMajorAllelesPlugin.java:20,24` | Modify 2 lines | the two filter param name contracts (Task 1) | +| `ApiCommonModel/.../params/variationParams.xml` | Add 2 filterParams + 4 threshold params | the doubled params (Task 2) | +| `ApiCommonModel/.../queries/variationQueries.xml` | Add 1 `processQuery` | plugin binding, 12 `wsColumn`s (Task 3) | +| `ApiCommonModel/.../variationQuestions.xml` | Add 1 question | 13 summary columns, 10 dynamic attributes, `uniq-value-params` (Task 3) | +| `ApiCommonModel/Model/lib/wdk/ontology/individuals.txt` | Append 1 row | category placement (Task 4) | + +No file gains new SQL. Both new filter params reuse `VariationVQ.SamplesMetadataByStudy` and +`SampleOntologyByStudy`. + +--- + +### Task 1: Rename the two filter param constants + +**Files:** +- Modify: `WSFPlugin/src/main/java/org/apidb/apicomplexa/wsfplugin/highspeedsnpsearch/FindMajorAllelesPlugin.java:20` and `:24` + +- [ ] **Step 1: Confirm the two lines and that nothing else references the old names** + +```bash +cd ~/workspaces/plasmodb/ApiCommonWebService +git branch --show-current +grep -rn "ngsSnp_strain_meta" WSFPlugin/src/main/java/ +``` + +Expected: branch `dnaseq-merge-experiments`, and exactly two hits, both in `FindMajorAllelesPlugin.java`: + +``` +:20: public static final String PARAM_STRAIN_FILTER_A = "ngsSnp_strain_meta_a"; +:24: public static final String PARAM_STRAIN_FILTER_B = "ngsSnp_strain_meta_m"; +``` + +If any other Java file references those strings, stop and report — the blast radius would be larger than the design assumed. + +- [ ] **Step 2: Make both changes** + +```java + public static final String PARAM_STRAIN_FILTER_A = "variation_sample_meta_a"; +``` +```java + public static final String PARAM_STRAIN_FILTER_B = "variation_sample_meta_b"; +``` + +Note `_m` becomes `_b`, not `_m`. Change nothing else in the file — in particular leave the +Set B strains handling alone, including the missing null check the design records as +deliberately out of scope. + +- [ ] **Step 3: Verify the diff is two lines and the old names are gone** + +```bash +cd ~/workspaces/plasmodb/ApiCommonWebService && git diff --stat && git diff +grep -rc "ngsSnp_strain_meta" WSFPlugin/src/main/java/ 2>/dev/null | grep -v ':0' || echo "old names gone" +``` + +Expected: `1 file changed, 2 insertions(+), 2 deletions(-)`, and `old names gone`. + +- [ ] **Step 4: Commit** + +```bash +cd ~/workspaces/plasmodb/ApiCommonWebService +git add WSFPlugin/src/main/java/org/apidb/apicomplexa/wsfplugin/highspeedsnpsearch/FindMajorAllelesPlugin.java +git commit -m "Rename the two sample-group param contracts for variation searches + +FindMajorAllelesPlugin serves VariationsByTwoIsolateGroups now. It extends +HighSpeedSnpSearchAbstractPlugin directly rather than FindPolymorphismsPlugin, +so these two names are its own constants and were not covered by the earlier +strain-filter rename. + +The odd _m becomes _b: nothing in the plugin distinguishes it beyond being the +second group, and its prompts already read Set B. With this, no snp-era param +name survives in any variation search." +``` + +- [ ] **Step 5: Build and install** + +```bash +cd ~/workspaces/agentic-veupath-dev && \ + ssh -o LogLevel=ERROR "$(python3 bin/resolve.py --profile profiles/plasmodb.yml --field host)" \ + "bash -lc 'source /var/www/jbrestel.plasmodb.org/etc/setenv && bld ApiCommonWebService'" +``` + +Expected: `BUILD SUCCESSFUL`, 1–2 minutes. + +- [ ] **Step 6: Reload and verify the installed jar** + +```bash +cd ~/workspaces/agentic-veupath-dev && bin/veup-build.sh plasmodb reload +``` + +Expected: `OK - Reloaded application at context path [/plasmo.jbrestel]`. + +```bash +ssh cedar "bash -lc 'J=/var/www/PlasmoDB/plasmo.jbrestel/webapp/WEB-INF/lib/api-common-websvc-wsfplugin-1.0.0.jar; \ + C=org/apidb/apicomplexa/wsfplugin/highspeedsnpsearch/FindMajorAllelesPlugin.class; \ + for s in variation_sample_meta_a variation_sample_meta_b ngsSnp_strain_meta_a ngsSnp_strain_meta_m; do \ + echo -n \"\$s=\"; unzip -p \$J \$C | strings | grep -c \$s; done'" +``` + +Expected: `variation_sample_meta_a=1`, `variation_sample_meta_b=1`, and **both old names `=0`**. +Remember jar entries are compressed, so `grep` over the lib directory would find nothing either +way — read the class out, as here. + +**A pass here is what makes Task 5 meaningful.** With an old string installed, the search fails +as a missing required parameter and the model looks wrong when it is fine. + +--- + +### Task 2: The two filter params and four threshold params + +**Files:** +- Modify: `ApiCommonModel/Model/lib/wdk/model/questions/params/variationParams.xml` + +- [ ] **Step 1: Confirm the EDA queries you are about to share already exist** + +No new SQL is written in this task; both filters point at the queries the one-group search uses. + +```bash +cd ~/workspaces/plasmodb/ApiCommonModel && python3 -c " +import xml.etree.ElementTree as T +r=T.parse('Model/lib/wdk/model/questions/params/variationParams.xml').getroot() +for qs in r.findall('querySet'): + print(qs.get('name'), [q.get('name') for q in qs.findall('sqlQuery')]) +" +``` + +Expected: `VariationVQ ['EdaSampleTableSuffix', 'SamplesMetadataByStudy', 'SampleOntologyByStudy', 'ChromosomeForVariations']`. + +- [ ] **Step 2: Add the two filter params** + +Inside the `variationParams` paramSet, after `chromosomeOptionalForVariations`: + +```xml + + + + + + + Select the first group of samples to compare. Use the sample characteristics + to narrow the group, or accept all samples for the organism you chose. + + + + + + Select the second group of samples to compare. It must differ from Set A; + comparing a group against itself returns nothing useful. + + +``` + +- [ ] **Step 3: Add the four threshold params** + +After the two filter params: + +```xml + + + + + + + + This parameter applies to the Set A aligned sample sequences. When a Set A + locus has a major allele frequency greater than or equal to this value, it + will be compared to the equivalent locus in Set B samples. Note that 100% is + permissible and is the most stringent setting, since the search first + identifies an allele in this set and then compares it with the allele in + Set B. See the Description below the Get Answer button for more. + + + \d\d?|100 + + + + + This parameter applies to the aligned sample sequences of Set B. When a Set B + locus has a major allele frequency greater than or equal to this value, it + will be compared to the equivalent locus in Set A samples. Note that 100% is + permissible, since the search first identifies loci from Set A and then + compares them with loci from Set B. See the Description below the Get Answer + button for more. + + + \d\d?|100 + + + + + This parameter applies to the Set B aligned sample sequences. At any given + nucleotide position, some samples in Set B may not have data supporting a + call because the Read Frequency Threshold was not met. This defines the + fraction of Set B samples that must have a base call before a locus is + returned for that position, based on the remaining samples that do have data. + See the Description below for more information. + + + \d\d?|100 + + + + + + This parameter applies to the sequencing reads of individual samples in Set B + and defines a stringency for data supporting a variant call between a sample + and the reference genome (Organism). Each nucleotide position of each sample + is compared to the reference genome and a call is made if the portion of the + sample's aligned reads that support the variant is above the Read Frequency + Threshold (RFT). Find high quality haploid variants with 80% RFT or + heterozygous diploid/aneuploid variants with 40%. See the Description below + for more. + + + + 80% + 80 + + + 60% + 60 + + + 40% + 40 + + + 20% + 20 + + + +``` + +Note `MinPercentIsolateCallsTwo`'s prompt has been harmonised with `MinPercentIsolateCalls`' +wording ("Percent samples with a base call >= ") rather than kept as the original's terser "Min +percent isolates with calls >= ". The design flagged this as the implementer's call; the two +prompts sit side by side in one form, so matching them is the kinder choice. **The param name is +unchanged** — only the prompt. + +- [ ] **Step 4: Verify the XML parses, the regexes survived, and nothing shares a name** + +```bash +cd ~/workspaces/plasmodb/ApiCommonModel && python3 - <<'PY' +import xml.etree.ElementTree as T, collections +r = T.parse('Model/lib/wdk/model/questions/params/variationParams.xml').getroot() +ps = r.find('paramSet') +names = [c.get('name') for c in ps] +print('params:', names) +dupes = [n for n,c in collections.Counter(names).items() if c > 1] +print('duplicate names:', dupes or 'none') +for sp in ps.iter('stringParam'): + rx = sp.find('regex') + print(sp.get('name'), '->', repr(rx.text if rx is not None else None)) +for e in ps.iter('enumParam'): + print(e.get('name'), 'internals:', [i.find('internal').text for i in e.iter('enumValue')]) +PY +``` + +Expected: thirteen params, **no duplicates**, the three `MinPercent*` regexes printing +`'\\d\\d?|100'` (and the pre-existing `MinPercentMinorAlleles` printing `'\\d\\d?'`), and both +enum params listing internals `['80', '60', '40', '20']`. A duplicate name here would mean you +redefined a param the one-group search already provides. + +- [ ] **Step 5: Build to prove the model still loads** + +```bash +cd ~/workspaces/agentic-veupath-dev && bin/veup-build.sh plasmodb wb model +``` + +Expected: completes with no `WdkModelException`. Unreferenced params are legal, so this only +proves the definitions resolve — Task 3 is what exercises them. + +```bash +ssh cedar 'bash -lc "source /var/www/jbrestel.plasmodb.org/etc/setenv && wdkXml -model PlasmoDB"' \ + | grep -E "name='(variation_sample_meta_a|variation_sample_meta_b|MinPercentMajorAlleles|MinPercentMajorAllelesTwo|MinPercentIsolateCallsTwo|ReadFrequencyPercentTwo)'" +``` + +Expected: six matching lines, one per new param, each prefixed with its Java class +(`FilterParamNew`, `StringParam`, `EnumParam`). Note the **single** quotes. + +- [ ] **Step 6: Commit** + +```bash +cd ~/workspaces/plasmodb/ApiCommonModel +git add Model/lib/wdk/model/questions/params/variationParams.xml +git commit -m "Add the two-group sample filters and four thresholds + +Both filters reuse the one-group search's EDA queries; no new SQL. WDK clones a +dependent param's queries per param, which is how the snp original ran two +groups off one query definition. + +No minSelectedCount on either group, deliberately: comparing major alleles +between two groups of one is meaningful, unlike polymorphism within one, and +the snp original set no minimum. + +Only Set B gets 'Two' threshold variants. Set A reuses the unsuffixed +ReadFrequencyPercent and MinPercentIsolateCalls because those are the plugin's +own Set A constants." +``` + +--- + +### Task 3: The query and the question + +**Files:** +- Modify: `ApiCommonModel/Model/lib/wdk/model/questions/queries/variationQueries.xml` +- Modify: `ApiCommonModel/Model/lib/wdk/model/questions/variationQuestions.xml` + +- [ ] **Step 1: Confirm the plugin's ten required params and twelve columns** + +```bash +cd ~/workspaces/plasmodb/ApiCommonWebService +sed -n '18,60p' WSFPlugin/src/main/java/org/apidb/apicomplexa/wsfplugin/highspeedsnpsearch/FindMajorAllelesPlugin.java +``` + +Expected: `PARAM_STRAIN_FILTER_A`/`_B` now reading `variation_sample_meta_a`/`_b` (Task 1), the +Set A constants naming the **unsuffixed** `ReadFrequencyPercent` and `MinPercentIsolateCalls`, +`getRequiredParameterNames()` listing ten, and `getColumns()` listing twelve. If Set A's +constants name suffixed params, stop — the plan's param wiring would be wrong. + +- [ ] **Step 2: Add the process query** + +In `variationQueries.xml`, inside the `VariationsBy` querySet, after `VariationsByGeneIds`: + +```xml + + + + + + + + The Organism defines the species identity of the samples and the genome + against which each sample's variants were called. + + + + + + + + + + + + + + + + + + + + + + + + + + + + +``` + +- [ ] **Step 3: Add the question** + +In `variationQuestions.xml`, after `VariationsByGeneIds`: + +```xml + + + + + + + + + + + + + +
+ + Each sample's sequencing reads are aligned to the reference genome (Organism) + and variants are recorded for each sample based on the Read Frequency + Threshold. Then, scanning locations across the samples in Set A and Set B + separately, the major allele of each set is recorded where it meets that + set's major allele frequency and percent samples with a base call. A location + is returned when the two sets' major alleles differ. + +

Choosing the two groups: Set A and Set B must differ. Use the + sample characteristics to define each group, for example samples from two + different countries, or two different host phenotypes.

+ +

Major allele frequency: Among the qualifying calls at a location + within one set, the major allele frequency is the percent carrying the most + common allele. Unlike the within-group searches, 100% is permissible here and + is the most stringent setting: the search identifies each set's major allele + first and then compares the two, so demanding unanimity within a set is a + sharper test rather than an impossible one. Lower the threshold to return + more locations.

+ +

Read frequency threshold: An allele is called for a sample at a + location if that fraction of the sample's aligned reads support it. Each set + has its own threshold.

+ +

Percent samples with a base call: A location is only considered + within a set if this fraction of that set's samples have a qualifying call + there.

+ ]]> + + + + + + + + + + + + + + + + + + + variation_sample_meta_a + variation_sample_meta_b + + + +``` + +- [ ] **Step 4: Verify both files parse and the question carries all three required blocks** + +```bash +cd ~/workspaces/plasmodb/ApiCommonModel && python3 - <<'PY' +import xml.etree.ElementTree as T +r = T.parse('Model/lib/wdk/model/questions/queries/variationQueries.xml').getroot() +s = r.find('querySet'); print('queries:', [c.get('name') for c in s]) +pq = [c for c in s if c.get('name') == 'VariationsByTwoIsolateGroups'][0] +print('paramRefs:', len(pq.findall('paramRef')), 'wsColumns:', len(pq.findall('wsColumn'))) +r2 = T.parse('Model/lib/wdk/model/questions/variationQuestions.xml').getroot() +print('questions:', [q.get('name') for q in r2.find('questionSet')]) +q = [x for x in r2.iter('question') if x.get('name') == 'VariationsByTwoIsolateGroups'][0] +print('dynAttrs:', len(list(q.iter('columnAttribute')))) +print('summary cols:', len(q.find('attributesList').get('summary').split(','))) +pl = [p for p in q.findall('propertyList') if p.get('name') == 'uniq-value-params'] +print('uniq-value-params:', [v.text for v in pl[0]] if pl else 'MISSING') +PY +``` + +Expected: five queries and five questions; **11 paramRefs** (organism, suffix, wsPath, plus +four per set) and **12 wsColumns**; **10** dynamic attributes; **13** summary columns; and +`uniq-value-params: ['variation_sample_meta_a', 'variation_sample_meta_b']`. `MISSING` there is +the failure this whole task is most likely to produce. + +- [ ] **Step 5: Build and confirm all five searches are in the assembled model** + +```bash +cd ~/workspaces/agentic-veupath-dev && bin/veup-build.sh plasmodb wb model +``` + +Expected: completes with no `WdkModelException`. + +```bash +ssh cedar 'bash -lc "source /var/www/jbrestel.plasmodb.org/etc/setenv && wdkXml -model PlasmoDB"' \ + | grep -oE "VariationsBy\.[A-Za-z]+:" | sort -u +``` + +Expected exactly five: `VariationBySourceId`, `VariationsByGeneIds`, `VariationsByIsolateGroup`, +`VariationsByLocation`, `VariationsByTwoIsolateGroups`. + +- [ ] **Step 6: Commit** + +```bash +cd ~/workspaces/plasmodb/ApiCommonModel +git add Model/lib/wdk/model/questions/queries/variationQueries.xml \ + Model/lib/wdk/model/questions/variationQuestions.xml +git commit -m "Add the VariationsByTwoIsolateGroups search + +Twelve wsColumns and an 11-column results file, both dictated by +FindMajorAllelesPlugin. Params ordered organism, then Set A, then Set B, with +prompt overrides so the two sets' thresholds are distinguishable in the form. + +Carries the uniq-value-params propertyList forbidding Set A = Set B. Its +enforcement is client-side, so no build or service check can detect its +absence; only the rendered form can." +``` + +--- + +### Task 4: Category ontology row + +**Files:** +- Modify: `ApiCommonModel/Model/lib/wdk/ontology/individuals.txt` + +- [ ] **Step 1: Append the row by substitution** + +```bash +cd ~/workspaces/plasmodb/ApiCommonModel +grep "VariationQuestions.VariationBySourceId" Model/lib/wdk/ontology/individuals.txt \ + | sed "s/VariationBySourceId/VariationsByTwoIsolateGroups/g" \ + >> Model/lib/wdk/ontology/individuals.txt +``` + +- [ ] **Step 2: Verify field counts match across all five rows** + +```bash +cd ~/workspaces/plasmodb/ApiCommonModel +awk -F'\t' '/VariationQuestions.Variation/ {print NF" fields "$6}' Model/lib/wdk/ontology/individuals.txt +``` + +Expected: five lines, all `14 fields`, the fifth naming +`VariationQuestions.VariationsByTwoIsolateGroups`. A differing count means a shifted column and +a silently misfiled search — stop rather than patching by hand. + +- [ ] **Step 3: Build with `wb ontology`, NOT `wb model`** + +```bash +cd ~/workspaces/agentic-veupath-dev && bin/veup-build.sh plasmodb wb ontology +``` + +Expected: completes, reporting `categories_merged.owl` saved. `wb model` here leaves the OWL +stale and the search uncategorized, **with no error anywhere**. + +- [ ] **Step 4: Prove the OWL has it under the right parent** + +```bash +ssh cedar 'grep -A3 "individuals.owl#VariationRecordClasses.VariationRecordClass.VariationQuestions.VariationsByTwoIsolateGroups\"" \ + /var/www/jbrestel.plasmodb.org/gus_home/lib/wdk/ontology/categories_merged.owl | grep -c topic_0199' +``` + +Expected: `1`. + +- [ ] **Step 5: Commit** + +```bash +cd ~/workspaces/plasmodb/ApiCommonModel +git add Model/lib/wdk/ontology/individuals.txt +git commit -m "Categorize VariationsByTwoIsolateGroups + +Same placement as the other four variation searches: parent topic_0199, +targetType search, menu + webservice scopes. Completes the five-search port." +``` + +--- + +### Task 5: Browser verification + +**Files:** none. + +- [ ] **Step 1: Mark the logs** + +```bash +cd ~/workspaces/agentic-veupath-dev && bin/veup-logs.sh plasmodb mark twogroups +``` + +- [ ] **Step 2: Open the app and confirm the origin** + +Load `https://jbrestel.plasmodb.org/a/app` (it redirects to the webapp context), then: + +```javascript +({origin: window.location.origin, base: window.location.pathname.split('/')[1]}) +``` + +Expected: origin `https://jbrestel.plasmodb.org`, base `plasmo.jbrestel`. If origin reads +`https://veupathdb.org`, **stop** — the tab bounced to autologin and relative fetches answer for +production. Build all paths from the base you actually got. + +`computer:screenshot` fails on this instance (`Script injection timed out`), including on +known-good pages. Use `javascript_tool` and the service endpoints. + +- [ ] **Step 3: Confirm all five searches are registered** + +```javascript +const b = window.location.pathname.split('/')[1]; +fetch(`/${b}/service/record-types/variation`).then(r=>r.json()) + .then(d=>d.searches.map(s=>s.fullName).filter(n=>n.startsWith('VariationQuestions'))) +``` + +Expected: all five, including `VariationQuestions.VariationsByTwoIsolateGroups`. + +- [ ] **Step 4: Confirm the form's shape and both filters populate** + +```javascript +const b = window.location.pathname.split('/')[1]; +const q = await fetch(`/${b}/service/record-types/variation/searches/VariationsByTwoIsolateGroups?expandParams=true`).then(r=>r.json()); +const ps = q.searchData.parameters; +JSON.stringify({ + names: ps.map(p=>p.name), + prompts: ps.map(p=>p.displayName), + filters: ps.filter(p=>p.type==='filter').map(p=>({name:p.name, nodes:p.ontology.length, min:p.minSelectedCount})) +}, null, 1) +``` + +Expected: eleven params; the two filters are `variation_sample_meta_a` and +`variation_sample_meta_b`, each with **27** ontology nodes and **no** `minSelectedCount`; and +the Set A / Set B prompts are distinct (not two identical "Read frequency threshold" labels). + +- [ ] **Step 5: The disjoint-groups run — the search's definition** + +`country` splits the Pf samples cleanly: 147 French Guiana, 69 Senegal. Run with Set A = +Senegal, Set B = French Guiana, thresholds at their defaults (80% RFT, 80 major allele, 20 +percent called). The `country` ontology term is `VAR_8e68b3e5`; a filter value looks like: + +```javascript +JSON.stringify({filters:[{field:"VAR_8e68b3e5", type:"string", isRange:false, + value:["Senegal"], includeUnknown:false}]}) +``` + +Submit via `/reports/standard` requesting attributes +`["primary_key","MajorAlleleA","MajorAlleleB"]`, then assert: + +```javascript +const bad = records.filter(r => r.attributes.MajorAlleleA === r.attributes.MajorAlleleB); +({total: meta.totalCount, sampled: records.length, violations: bad.length}) +``` + +Expected: a non-empty result set and **`violations: 0`**. `MajorAlleleA != MajorAlleleB` *is* +the search's definition, so a single violation means the comparison is broken — this is a +stronger assertion than any row count. + +- [ ] **Step 6: The symmetry check** + +Swap the groups: Set A = French Guiana, Set B = Senegal. Expect an **identical `totalCount`**, +because "the two major alleles disagree" is symmetric. A differing count means Set B's +thresholds are not applied the way Set A's are — exactly the copy-paste asymmetry that doubled +params invite. + +Caveat to record in the report: this tests the *threshold* plumbing. It would also pass +trivially if both sets read the same `readFreq` files, which on this haploid site they do. + +- [ ] **Step 7: The `uniq-value-params` check — browser only** + +In the rendered form, set both groups to the same value (Senegal in each). The form should +refuse to submit. If it submits, the property did not take effect; report it rather than working +around it, since nothing else can detect this. + +- [ ] **Step 8: Confirm an ID resolves** + +Take one returned ID (form `Variant__`), navigate to +`//app/record/variation/`, and confirm the record page renders without error. + +- [ ] **Step 9: Category tree** + +```javascript +const b = window.location.pathname.split('/')[1]; +const c = await fetch(`/${b}/service/ontologies/Categories`).then(r=>r.json()); +const out=[]; +(function walk(n,parent){const p=n.properties||{};const nm=(p.name||[])[0]; + if(nm && nm.startsWith('VariationQuestions')) out.push({name:nm.replace('VariationQuestions.',''), parent}); + (n.children||[]).forEach(ch=>walk(ch,(p['EuPathDB alternative term']||p.label||[])[0]||parent)); +})(c.tree,'ROOT'); +JSON.stringify(out,null,1) +``` + +Expected: all five searches, each with parent `"Genetic variation"`. + +- [ ] **Step 10: Logs** + +```bash +cd ~/workspaces/agentic-veupath-dev && bin/veup-logs.sh plasmodb since twogroups --quiet +cd ~/workspaces/agentic-veupath-dev && bin/veup-logs.sh plasmodb since twogroups 2>&1 \ + | grep -oE "hsssGenerate[A-Za-z]+|findMajorAlleles" | sort | uniq -c +``` + +Expected: error logs `silent:`, and `hsssGenerateMajorAllelesScript` appearing once per run. + +**Any ERROR lines caused by your own malformed service requests must be reported as +self-inflicted, not as "the logs were clean."** + +- [ ] **Step 11: Report** + +Report: the five registered searches; the eleven params with their prompts and the two filters' +node counts; for both runs the total count and the violation count; whether the form refused +A = B; one ID that resolved; the category-tree parents; and the log verdict distinguishing +self-inflicted errors from real ones. + +--- + +## Out of scope + +- **`NgsSnpsByTwoIsolateGroupsWiz`** — a sixth search (PlasmoDB/UniDB only) driven by the + `*_wiz` params, which sit in the same commented-out region as the params this plan replaces. + Whether the wizard flow is still wanted is a product question. **With this plan the + five-search port is complete.** +- **Deleting the dead snp XML.** After this, `snpParams.xml` and the commented-out snp regions + of `sharedParams.xml` have no remaining consumer, but removal has its own blast radius + (`recordParams.xml`, `spanQuestions.xml`, `SnpsBySpanLogic`). +- **The Set B null-check asymmetry** in `FindMajorAllelesPlugin`. +- **`ReadFrequencyPercent` as a functional parameter.** All four `readFreq*` directories hold + identical data on haploid organisms; both read-frequency params select their directory + correctly, and changing either will not change results. Not a defect in this search. From 9f6aae1cebc218472fc9e76b91f073bf5d84e015 Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Wed, 5 Aug 2026 16:48:53 -0400 Subject: [PATCH 54/98] Add the two-group sample filters and four thresholds Both filters reuse the one-group search's EDA queries; no new SQL. WDK clones a dependent param's queries per param, which is how the snp original ran two groups off one query definition. No minSelectedCount on either group, deliberately: comparing major alleles between two groups of one is meaningful, unlike polymorphism within one, and the snp original set no minimum. Only Set B gets 'Two' threshold variants. Set A reuses the unsuffixed ReadFrequencyPercent and MinPercentIsolateCalls because those are the plugin's own Set A constants. --- .../questions/params/variationParams.xml | 132 ++++++++++++++++++ 1 file changed, 132 insertions(+) diff --git a/Model/lib/wdk/model/questions/params/variationParams.xml b/Model/lib/wdk/model/questions/params/variationParams.xml index d119272d45..3f9de86881 100644 --- a/Model/lib/wdk/model/questions/params/variationParams.xml +++ b/Model/lib/wdk/model/questions/params/variationParams.xml @@ -196,6 +196,138 @@ + + + + + + + Select the first group of samples to compare. Use the sample characteristics + to narrow the group, or accept all samples for the organism you chose. + + + + + + Select the second group of samples to compare. It must differ from Set A; + comparing a group against itself returns nothing useful. + + + + + + + + + + + This parameter applies to the Set A aligned sample sequences. When a Set A + locus has a major allele frequency greater than or equal to this value, it + will be compared to the equivalent locus in Set B samples. Note that 100% is + permissible and is the most stringent setting, since the search first + identifies an allele in this set and then compares it with the allele in + Set B. See the Description below the Get Answer button for more. + + + \d\d?|100 + + + + + This parameter applies to the aligned sample sequences of Set B. When a Set B + locus has a major allele frequency greater than or equal to this value, it + will be compared to the equivalent locus in Set A samples. Note that 100% is + permissible, since the search first identifies loci from Set A and then + compares them with loci from Set B. See the Description below the Get Answer + button for more. + + + \d\d?|100 + + + + + This parameter applies to the Set B aligned sample sequences. At any given + nucleotide position, some samples in Set B may not have data supporting a + call because the Read Frequency Threshold was not met. This defines the + fraction of Set B samples that must have a base call before a locus is + returned for that position, based on the remaining samples that do have data. + See the Description below for more information. + + + \d\d?|100 + + + + + + This parameter applies to the sequencing reads of individual samples in Set B + and defines a stringency for data supporting a variant call between a sample + and the reference genome (Organism). Each nucleotide position of each sample + is compared to the reference genome and a call is made if the portion of the + sample's aligned reads that support the variant is above the Read Frequency + Threshold (RFT). Find high quality haploid variants with 80% RFT or + heterozygous diploid/aneuploid variants with 40%. See the Description below + for more. + + + + 80% + 80 + + + 60% + 60 + + + 40% + 40 + + + 20% + 20 + + + + + + + + + + + The Organism defines the species identity of the samples and the genome + against which each sample's variants were called. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/Model/lib/wdk/model/questions/variationQuestions.xml b/Model/lib/wdk/model/questions/variationQuestions.xml index c0f83f9f92..1813f6fe9b 100644 --- a/Model/lib/wdk/model/questions/variationQuestions.xml +++ b/Model/lib/wdk/model/questions/variationQuestions.xml @@ -295,6 +295,90 @@ + + + + + + + + + + + + + +
+ + Each sample's sequencing reads are aligned to the reference genome (Organism) + and variants are recorded for each sample based on the Read Frequency + Threshold. Then, scanning locations across the samples in Set A and Set B + separately, the major allele of each set is recorded where it meets that + set's major allele frequency and percent samples with a base call. A location + is returned when the two sets' major alleles differ. + +

Choosing the two groups: Set A and Set B must differ. Use the + sample characteristics to define each group, for example samples from two + different countries, or two different host phenotypes.

+ +

Major allele frequency: Among the qualifying calls at a location + within one set, the major allele frequency is the percent carrying the most + common allele. Unlike the within-group searches, 100% is permissible here and + is the most stringent setting: the search identifies each set's major allele + first and then compares the two, so demanding unanimity within a set is a + sharper test rather than an impossible one. Lower the threshold to return + more locations.

+ +

Read frequency threshold: An allele is called for a sample at a + location if that fraction of the sample's aligned reads support it. Each set + has its own threshold.

+ +

Percent samples with a base call: A location is only considered + within a set if this fraction of that set's samples have a qualifying call + there.

+ ]]> + + + + + + + + + + + + + + + + + + + variation_sample_meta_a + variation_sample_meta_b + + + + From e13137562c610e8583216d2f2c4b816321d51c71 Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Wed, 5 Aug 2026 16:56:54 -0400 Subject: [PATCH 57/98] Categorize VariationsByTwoIsolateGroups Same placement as the other four variation searches: parent topic_0199, targetType search, menu + webservice scopes. Completes the five-search port. --- Model/lib/wdk/ontology/individuals.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/Model/lib/wdk/ontology/individuals.txt b/Model/lib/wdk/ontology/individuals.txt index 9873ea59dd..a9d3c35e13 100644 --- a/Model/lib/wdk/ontology/individuals.txt +++ b/Model/lib/wdk/ontology/individuals.txt @@ -1159,3 +1159,4 @@ VariationRecordClasses.VariationRecordClass.VariationQuestions.VariationBySource VariationRecordClasses.VariationRecordClass.VariationQuestions.VariationsByIsolateGroup http://edamontology.org/topic_0199 Genetic Variation VariationRecordClasses.VariationRecordClass search VariationQuestions.VariationsByIsolateGroup menu webservice VariationRecordClasses.VariationRecordClass.VariationQuestions.VariationsByLocation http://edamontology.org/topic_0199 Genetic Variation VariationRecordClasses.VariationRecordClass search VariationQuestions.VariationsByLocation menu webservice VariationRecordClasses.VariationRecordClass.VariationQuestions.VariationsByGeneIds http://edamontology.org/topic_0199 Genetic Variation VariationRecordClasses.VariationRecordClass search VariationQuestions.VariationsByGeneIds menu webservice +VariationRecordClasses.VariationRecordClass.VariationQuestions.VariationsByTwoIsolateGroups http://edamontology.org/topic_0199 Genetic Variation VariationRecordClasses.VariationRecordClass search VariationQuestions.VariationsByTwoIsolateGroups menu webservice From f41a71b9b04c19070351ae8c7225292f160078d8 Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Thu, 6 Aug 2026 11:53:34 -0400 Subject: [PATCH 58/98] Design the port of the four remaining Genetic Variation searches GenesByNgsSnps, SequencesByPloidy, GenesByCopyNumber and GenesByCopyNumberComparison are the last consumers of two dead subsystems: the snp paramSet (absent from the assembled model) and the PAN provenance graph. The root cause for the three CNV searches is not the sample param but a dead join: study.Input has 0 rows, so webready.PANIO_p is empty, so GeneCopyNumbers_p and ChrCopyNumbers_p are empty. Both organism scoping and sample identity have to be re-sourced. Verified replacements: org identity from the attribute tables, and sample identity from the protocolappnode name minus its _GeneCNV/_Ploidy suffix, which matches an EDA sample stable id 441/441 and 452/452. Corrected tables land transitionally in apidbtuning (buildable now) and in webready/*.psql (next workflow run), with an explicit one-release sunset so the transitional copies do not become permanent. Co-Authored-By: Claude Opus 5 --- ...-genetic-variation-searches-port-design.md | 485 ++++++++++++++++++ 1 file changed, 485 insertions(+) create mode 100644 docs/superpowers/specs/2026-08-06-genetic-variation-searches-port-design.md diff --git a/docs/superpowers/specs/2026-08-06-genetic-variation-searches-port-design.md b/docs/superpowers/specs/2026-08-06-genetic-variation-searches-port-design.md new file mode 100644 index 0000000000..84e490af8e --- /dev/null +++ b/docs/superpowers/specs/2026-08-06-genetic-variation-searches-port-design.md @@ -0,0 +1,485 @@ +# Genetic Variation searches port — design + +**Date:** 2026-08-06 +**Status:** designed +**Scope:** Port the four remaining Genetic Variation searches onto the merged-dnaseq / +EDA world: `GenesByNgsSnps`, `SequencesByPloidy`, `GenesByCopyNumber`, +`GenesByCopyNumberComparison`. +**Repo:** `ApiCommonModel` only, branch `dnaseq-merge-experiments`. +**Builds on:** `2026-08-05-hsss-variation-plumbing-design.md` (ApiCommonWebService) and the +five `variation*` search specs. Those established the organism → EDA-study → samples-filter +pattern this spec reuses. + +## 1. Purpose + +Five variation searches now work against the merged dnaseq EDA studies. Four searches +remain, and they are the last consumers of two dead subsystems: + +- the **snp** `paramSet`, which is absent from the assembled model (its `` sits + inside a commented block in `apiCommonModel.xml`); and +- the **PAN provenance graph** (`study.Input` → `webready.PANIO_p`), which is no longer + loaded. + +The four searches split into two families with unrelated blockers, but they share the +sample-selection pattern and are categorized together, so they are specified together and +built in three phases (§2). + +### 1.1 Current state, verified + +Against the assembled model and `unidb_shu_a`: + +| Search | model XML | ontology row | blocker | +|---|---|---|---| +| `GenesByNgsSnps` | **commented out** (question + `processQuery`) | live, orphaned (`individuals.txt:99`) | referenced `snpParams.*`; plugin now fixed | +| `GenesByCopyNumber` | live | `##` commented (`:100`) | reads empty `webready.GeneCopyNumbers_p` | +| `GenesByCopyNumberComparison` | live | `##` commented (`:101`) | same | +| `SequencesByPloidy` | live | `##` commented (`:138`) | reads dead `CNV_strain` / `organismSinglePickCnv` | + +Three of the four are *live but dark*: they build, and nothing surfaces them. + +### 1.2 Why the CNV tables are empty — root cause + +`webready.GeneCopyNumbers_p` and `ChrCopyNumbers_p` both inner-join `webready.PANIO_p`. +`PANIO_p` is empty because it inner-joins `study.Input`, and **`study.Input` has 0 rows** +on this build (`study.Output` 1,356; `study.ProtocolApp` 1,356). So `input_pan_id` and +`output_pan_id` — which all three CNV searches use for *both* sample identity and organism +scoping — have no data behind them. + +This is not a dev-subset artifact. It means the CNV searches cannot be repaired by swapping +a param; the join they are built on has to be replaced. + +The underlying data is fine: `apidb.genecopynumber` 3,404,180 rows, `apidb.chrcopynumber` +4,936 rows. + +### 1.3 What replaces the PAN graph + +Everything the dead columns supplied is available elsewhere, and was verified rather than +assumed: + +| lost | replacement | evidence | +|---|---|---| +| organism scoping via `org_abbrev` on the tuning table | `org_abbrev` / `organism` / `taxon_id` from `webready.TranscriptAttributes_p` (69,397 rows) and `GenomicSeqAttributes_p` (8,348 rows) | **all** 3,389,444 gene-CNV rows find a match on `gene_na_feature_id` (fanning out to 3,404,180 result rows via multi-transcript genes); **all** 4,936 chr-CNV rows join on `na_sequence_id` | +| sample identity via `output_pan_id` | `study.protocolappnode.name` minus its `_GeneCNV` / `_Ploidy` suffix, which **is** an EDA sample stable ID | 441/441 gene-CNV and 452/452 chr-CNV names match exactly; per-organism 216/216 pfal3D7, 232/232 afumAf293, 4/4 tbruTREU927 | +| gene↔chr correlation via `input_pan_id` | `eda_sample_stable_id` + `na_sequence_id` | all 441 gene-CNV samples present among the 452 chr-CNV samples | + +## 2. Sequencing + +`GenesByNgsSnps` depends on none of the CNV work, so it ships first and the Jenkins halt +blocks only half the effort. + +| Phase | Work | Gate | +|---|---|---| +| **1** | `GenesByNgsSnps`: uncomment, repoint params, ontology, live QA | none — HSSS plumbing already deployed | +| **2** | Corrected CNV table SQL proven in the `jbrestel` schema; `apiTuningManager.xml` entries; `webready/*.psql` corrections | — | +| **HALT** | **John builds the tuning tables in Jenkins.** Nothing in phase 3 can be exercised before this. | | +| **3** | Three CNV searches: model XML, ontology, live QA | phase 2's tables | + +`jbrestel.*` is a throwaway proof of concept and must never appear in a commit. The +committed model names `apidbtuning.*` (§7). + +## 3. The corrected CNV tables + +Two logical tables, each defined in two places (§7 explains why): + +- transitional: `apidbtuning.GeneCopyNumbers`, `apidbtuning.ChrCopyNumbers` +- permanent: `webready.GeneCopyNumbers_p`, `webready.ChrCopyNumbers_p` + +### 3.1 `GeneCopyNumbers` + +```sql +SELECT DISTINCT + ta.project_id + , ta.org_abbrev + , ta.organism + , ta.taxon_id + , ta.source_id + , ta.gene_source_id + , regexp_replace(pan.name, '_GeneCNV$', '') AS eda_sample_stable_id + , gcn.haploid_number AS raw_estimate + , gcn.ref_copy_number AS ref_cn + , CASE WHEN (gcn.haploid_number < 0.01) THEN 0 + WHEN (0.01 < gcn.haploid_number AND gcn.haploid_number < 1.85) THEN 1 + ELSE round(gcn.haploid_number) END AS haploid_number + , ta.chromosome + , ta.na_sequence_id +FROM apidb.genecopynumber gcn +JOIN study.protocolappnode pan + ON pan.protocol_app_node_id = gcn.protocol_app_node_id +JOIN webready.transcriptattributes_p ta + ON ta.gene_na_feature_id = gcn.na_feature_id +WHERE ta.gene_type IN ('protein coding', 'protein coding gene') +``` + +### 3.2 `ChrCopyNumbers` + +```sql +SELECT DISTINCT + sa.project_id + , sa.org_abbrev + , sa.organism + , sa.taxon_id + , sa.source_id + , sa.na_sequence_id + , sa.chromosome + , ccn.chr_copy_number AS ploidy + , regexp_replace(pan.name, '_Ploidy$', '') AS eda_sample_stable_id +FROM apidb.chrcopynumber ccn +JOIN study.protocolappnode pan + ON pan.protocol_app_node_id = ccn.protocol_app_node_id +JOIN webready.genomicseqattributes_p sa + ON sa.na_sequence_id = ccn.na_sequence_id +WHERE sa.chromosome IS NOT NULL +``` + +### 3.3 Changes from today's definitions, and why + +1. **The `PANIO_p` join is gone**, along with `input_pan_id` / `output_pan_id`. §1.2. +2. **Organism identity comes from the attribute tables**, carrying `org_abbrev`, + `organism`, and `taxon_id`. Carrying all three is deliberate: the searches filter on + `organism` (§4.1) while the permanent partitioned table prunes on `org_abbrev`, and + nothing should have to translate between them at query time. +3. **New `eda_sample_stable_id`** — the join key the samples filterParam produces. Without + it every search would have to round-trip through `study.protocolappnode` and re-derive + the same string. +4. **The `strain` column is retired.** Its derivation, + `REGEXP_REPLACE(pan.name, '_[A-Za-z0-9]+ (.+)$', '')`, requires a **space** that these + names never contain, so it is a no-op and `strain` currently holds the un-stripped + `427_GeneCNV`. `eda_sample_stable_id` replaces it, correctly. The searches' `strains` + *output* column (the aggregated list of matching samples) is unaffected and keeps its + name — it is now aggregated from `eda_sample_stable_id`. +5. **The rounding CASE is preserved verbatim**, including its gap at exactly `0.01` (a + value that falls through to `round()` → `0`). No row currently has that value, so the + gap is inert; it is kept anyway because a port must not silently change data semantics. + Both columns are load-bearing: searches *filter* on the rounded `haploid_number` and + *display* medians of `raw_estimate`. +6. **`ChrCopyNumbers` sources from `GenomicSeqAttributes_p`**, not `TranscriptAttributes_p` + as today. Chromosome ploidy is sequence-level; reaching through the transcript table for + it was incidental. `SequencesByPloidy` already reads `GenomicSeqAttributes_p`, so this + also makes the table and its main consumer agree. + +### 3.4 Physical design + +| | transitional (`apidbtuning`) | permanent (`webready`) | +|---|---|---| +| partitioning | **none** | partitioned on `org_abbrev` (unchanged) | +| index | btree `(organism, eda_sample_stable_id)` | existing `_ix.psql`, updated to drop `input_pan_id` | + +The transitional table is unpartitioned on purpose. `apidbtuning` does support partitioned +tables (`allgeneproducts_p1118`, `profile_p1118` are `relkind='p'`), so this is a choice, +not a limitation: the whole table is ~3.4M rows, every search filters to one organism, and +a composite btree serves that nearly as well as a prune. Partitioning is build complexity +on an object scheduled for deletion in one release (§7). + +The two `_ix.psql` files must drop `input_pan_id` from the index definition, since the +column is gone. + +## 4. Params + +### 4.1 CNV organism: a `queryRef` override, not a new param + +The three CNV searches keep using **`organismParams.organismSinglePick`**, overriding its +vocabulary per query: + +```xml + +``` + +This is the pattern the original `GenesByNgsSnps` used (`queryRef="organismVQ.withNgsSNPsTree"`). + +**It must not be a new param.** `variationParams.eda_sample_table_suffix` and the samples +filterParam both declare `dependedParamRef="organismParams.organismSinglePick"` and +interpolate `$$organismSinglePick$$`. A genuinely new organism param would leave that chain +pointing at a param the query no longer has, so the samples filter would silently never +scope to the chosen organism. + +`organismVQ.CNVDnaSeq` returns `internal` = **scientific name**, matching +`organismSinglePick`'s convention. The dead `organismVQ.CNV` returned +`string_agg(o.abbrev)` — a comma-joined list of abbreviations — which is a second reason +nothing downstream lined up. It lists organisms that actually have CNV rows, so an organism +that would return nothing is not selectable. + +### 4.2 Add + +| item | file | notes | +|---|---|---| +| `organismVQ.CNVDnaSeq` | `organismParams.xml` | §4.1 | +| `variationParams.cnv_sample_meta` | `variationParams.xml` | same `metadataQueryRef` / `backgroundQueryRef` / `ontologyQueryRef` and `dependedParamRef` as `variation_sample_meta`; prompt `Strain/Sample` | + +`cnv_sample_meta` deliberately has **no `minSelectedCount`**. `variation_sample_meta` +requires 2 because polymorphism within a group of one is undefined; a copy-number query on +a single sample is perfectly meaningful. + +It is a separate param rather than a reuse of `variation_sample_meta` because that name is +a contract with `FindPolymorphismsPlugin.getStrainFilterParamName()`, and because the two +differ in `minSelectedCount`. WDK clones a dependent param's queries per param, so sharing +the three query definitions costs nothing — the same arrangement `variation_sample_meta_a` +and `_b` already use. + +### 4.3 Retire + +| item | file | reason | +|---|---|---| +| `organismParams.organismSinglePickCnv` | `organismParams.xml` | only consumer is the CNV searches | +| `organismVQ.CNV` | `organismParams.xml` | returns **0 rows** — no datasource matches `%copynumbervariations_%`; CNV now rides `isolates`/`Dna_Seq` | +| `sharedParams.CNV_strain` | `sharedParams.xml` | reads tables that do not exist | +| `SharedVQ.CnvSamplesMetadataByOrganism` | `sharedParams.xml` | reads `apidbTuning.Metadata` — **table does not exist** | +| `SharedVQ.CnvMetadataSpecByOrganism` | `sharedParams.xml` | reads `apidbTuning.Ontology` — **table does not exist** | + +Before removing each, confirm no other consumer with a search of the assembled model, not a +`grep` of the source — `grep` matches inside comment blocks, which is how the snp import was +previously misread. + +### 4.4 Reuse unchanged + +All fourteen supporting params were verified live in the assembled model and need no work: + +- `geneParams`: `snp_class`, `occurrences_lower`, `occurrences_upper`, `dn_ds_ratio_lower`, + `dn_ds_ratio_upper`, `snp_density_lower`, `snp_density_upper`, `copyNumber`, `CNV_type`, + `operator`, `comparisonOperator`, `medianOrIndividual` +- `genomicParams`: `chrCopyNumber`, `medianOrIndividual` + +### 4.5 `GenesByNgsSnps` param repointing + +`snpParams` is absent from the assembled model, so every `snpParams.*` ref must move to its +`variationParams` twin — which exists precisely because of this (see the variation searches +spec, §4.4): + +| was | becomes | +|---|---| +| `snpParams.WebServicesPath` | `variationParams.WebServicesPath` | +| `snpParams.ReadFrequencyPercent` | `variationParams.ReadFrequencyPercent` | +| `snpParams.MinPercentMinorAlleles` | `variationParams.MinPercentMinorAlleles` | +| `snpParams.MinPercentIsolateCalls` | `variationParams.MinPercentIsolateCalls` | +| `snpParams.ngsSnp_strain_meta` | `variationParams.variation_sample_meta` | +| `organismSinglePick` + `queryRef="organismVQ.withNgsSNPsTree"` | plain `organismSinglePick` (drop the override) | + +The strain-filter repoint is not a preference: `FindGenesWithSnpCharsPlugin extends +FindPolymorphismsPlugin`, so it inherits `getStrainFilterParamName()` returning +`variation_sample_meta`. The XML param name must equal that string or the search fails at +run time, not build time. + +`organismVQ.withNgsSNPsTree` reads `apidbtuning.snpstrains`, which does not exist in this +build; plain `organismSinglePick` is what the five working variation searches use. + +## 5. Queries + +### 5.1 `GeneId.GenesByNgsSnps` (`geneQueries.xml`) + +Uncomment the `processQuery`; apply §4.5. Everything else stands: + +- the three `postCacheUpdateSql` blocks read `webready.TranscriptAttributes_p` (healthy) and + `%%PARTITION_KEYS%%`, which the webapp's partition layer substitutes at run time; +- the eleven `wsColumn` declarations are the plugin's own output contract and are unchanged; +- `processName` stays `…highspeedsnpsearch.FindGenesWithSnpCharsPlugin`. + +Drop the `testParamValues` blocks naming `ngsSnp_strain_meta` — the param no longer exists +under that name, and their values are old strain names rather than EDA sample stable IDs. + +### 5.2 `GeneId.GenesByCopyNumber` and `GenesByCopyNumberComparison` (`geneQueries.xml`) + +Both share one `bySample` CTE; the edits are identical in each. + +| was | becomes | +|---|---| +| `FROM webready.GeneCopyNumbers_p g, webready.ChrCopyNumbers_p c` | `FROM apidbtuning.GeneCopyNumbers g, apidbtuning.ChrCopyNumbers c` | +| `c.output_pan_id IN ($$CNV_strain$$)` | `c.eda_sample_stable_id IN ($$cnv_sample_meta$$)` | +| `g.input_pan_id = c.input_pan_id` | `g.eda_sample_stable_id = c.eda_sample_stable_id` | +| `g.org_abbrev = $$organismSinglePickCnv$$` / `c.org_abbrev = …` | `g.organism = $$organismSinglePick$$` / `c.organism = …` | +| `g.strain` (in the `string_agg`) | `g.eda_sample_stable_id` | + +`g.na_sequence_id = c.na_sequence_id` is retained. Together with the sample equality it +reproduces exactly what the `input_pan_id` join meant: *the same sample's ploidy for the +chromosome this gene sits on*. + +The `medians` / `hit_medians` CTEs, the `percentile_cont` aggregates, the `CASE` gates on +`$$CNV_type$$` / `$$operator$$` / `$$medianOrIndividual$$` / `$$comparisonOperator$$`, and +the fifteen output columns are all unchanged. + +### 5.3 `SequenceIds.ByCopyNumber` (`genomicQueries.xml`) + +This one reads base tables today and never touched `ChrCopyNumbers_p`. Move it onto +`apidbtuning.ChrCopyNumbers` so the three CNV searches share one definition of what a +sample and a ploidy are, and so it stops carrying its own copy of the broken `strain` regex: + +| was | becomes | +|---|---| +| `FROM apidb.chrcopynumber ccn, study.protocolappnode pan, webready.GenomicSeqAttributes_p sa` | `FROM apidbtuning.ChrCopyNumbers c` | +| `ccn.protocol_app_node_id IN ($$CNV_strain$$)` | `c.eda_sample_stable_id IN ($$cnv_sample_meta$$)` | +| `string_agg(REGEXP_REPLACE(s.name, '_[A-Za-z0-9]+ (.+)$', ''), ', ' …)` | `string_agg(c.eda_sample_stable_id, ', ' …)` | +| *(no organism filter — it had none)* | `c.organism = $$organismSinglePick$$` | + +Adding the organism filter is a **behaviour fix, not a regression**: the query previously +constrained organism only implicitly, through `CNV_strain`'s organism-scoped vocabulary. With +the sample list now coming from an EDA study, the constraint must be explicit or a sample +name colliding across organisms would leak rows. + +## 6. Questions and categorization + +### 6.1 `GenesByNgsSnps` (`geneQuestions.xml`) + +Uncomment. Keep `displayName`, `shortDisplayName`, both project-scoped `` +blocks, the `attributesList`, and all eight `dynamicAttributes`. Remove the deprecated +`` blocks — the file's own comment marks them deprecated and +they encode a fixed organism list this search no longer needs. + +`searchCategory="Population Biology"` is retained. + +### 6.2 `GenesByCopyNumber` / `GenesByCopyNumberComparison` (`geneQuestions.xml`) + +Both have their `attributesList` **commented out**, so ten CNV columns are declared in +`dynamicAttributes` and never shown — the searches return bare gene rows. Restore it with +the CNV columns only: + +```xml + +``` + +The `sorting` attribute is **new** — the commented original carried none, and an unsorted +copy-number result is not useful. `median_haploid_hits desc` puts the highest-amplification +genes first, which is what the search is usually asked for. It is a judgment call, not a +restoration; change it freely if the biologists prefer another default. + +The commented original also named `gene_product`, `chromosome`, and `orthomcl_link`. All +three were verified to exist on `TranscriptRecordClass` (`transcriptRecord.xml:339`, `:327`, +`:869`), so their absence here is a **deliberate choice to keep the summary to what the +search is about**, not a workaround for missing attributes. Recorded so it is not +re-litigated as an oversight. + +The `_all` variants (`median_raw_all`, `median_haploid_all`, `median_ploidy_all`, +`median_gene_dose_all`) stay declared but out of the default summary, as in the original. + +### 6.3 `SequencesByPloidy` (`genomicQuestions.xml`) + +Params only (§4, §5.3). `attributesList`, `dynamicAttributes`, `summary`, and `description` +are unchanged. + +### 6.4 `individuals.txt` + +| line | action | +|---|---| +| 99 `GeneQuestions.GenesByNgsSnps` | already live and correct — becomes non-orphaned once §6.1 lands | +| 100 `GeneQuestions.GenesByCopyNumber` | uncomment (`##` → live) | +| 101 `GeneQuestions.GenesByCopyNumberComparison` | uncomment | +| 138 `GenomicSequenceQuestions.SequencesByPloidy` | uncomment **and** move `topic_0219`/`Curation and Annotation` → `topic_0199`/`Genetic Variation` | + +All five variation searches sit under `topic_0199`, so this puts all eight dnaseq searches +in one menu section. + +Build target is **`wb ontology`**, not `wb model` — categorization changed. `wb ontology` is +a superset of `wb model`, so it is the single call for both phases. + +## 7. The transitional tables, and their sunset + +The webready tables are built per organism by the workflow, which has already completed for +this release. Rather than block on a workflow re-run, the corrected definitions land in two +places: + +1. **`apiTuningManager.xml`** gains `GeneCopyNumbers` and `ChrCopyNumbers` tuning tables + (§3), buildable now via Jenkins. The committed model XML references these. +2. **`webready/orgSpecific/*.psql`** (4 files: two definitions, two indexes) are corrected + to match, taking effect at the next workflow run. + +Both are written from SQL already proven in the `jbrestel` schema (§2), so they are two +copies of a verified definition rather than two guesses. + +**Sunset, one release later** — this is a scheduled deletion, not an aspiration: + +- delete the two `` entries from `apiTuningManager.xml`; +- repoint three queries (§5.2 ×2, §5.3) from `apidbtuning.X` to `webready.X_p`; +- change the organism predicate from `organism = …` back to `org_abbrev = …` **only if** + partition pruning measures better than the composite index — the `organism` column is + carried on the webready table too (§3.3), so no schema change is required either way. + +Each of the two definition sites must carry a comment naming the other and naming this +section, so an edit to one cannot silently diverge from the other during the release they +coexist. + +## 8. Verification + +### 8.1 Phase 1 — `GenesByNgsSnps` + +1. `bin/veup-build.sh plasmodb wb ontology` +2. `/service/record-types/transcript` → confirm `GenesByNgsSnps` in `searches`. This, not + the category tree, is the source of truth for "does this site have the search" — the + tree is not project-filtered. +3. `/service/ontologies/Categories` → confirm it sits under the `Genetic Variation` node. +4. Run it live in the app with a small sample set; confirm non-zero results and populated + SNP-characteristic columns. This also exercises the HSSS `/dnaseq` directory and the + `Variant_` ID work end to end. + +### 8.2 Phase 2 — CNV tables + +1. Build both tables in the `jbrestel` schema from §3's SQL. +2. Parity checks against the base tables: + - every one of `apidb.genecopynumber`'s 3,389,444 rows is represented — none dropped by + the join. The join fans out to 3,404,180 rows because a gene may have several + transcripts, and `SELECT DISTINCT` then collapses rows identical across all selected + columns, so the final count sits at or below 3,404,180 and **above** 3,389,444 is not + expected. Assert no base row is lost, rather than asserting an exact total. + - the `gene_type IN ('protein coding', 'protein coding gene')` filter currently removes + **nothing** (3,404,180 both with and without it). It is retained because it is the + original definition's semantics, but a parity check must not treat it as load-bearing. + - chr-CNV = 4,936 rows exactly (no fan-out — the join is on `na_sequence_id`). + - `eda_sample_stable_id` distinct counts 441 (gene) / 452 (chr), with all 441 present + among the 452. +3. Spot-check the rounding CASE: rows below `0.01`, between `0.01` and `1.85`, and above. +4. `EXPLAIN (ANALYZE)` the `bySample` CTE at 10 / 50 / all samples and confirm the index is + used. Baselines measured on base tables for pfal3D7: 218 ms / 986 ms / 4.1 s. + +### 8.3 Phase 3 — CNV searches + +Live QA is only possible after the Jenkins build. Before it: + +``` +ssh 'bash -lc "source /../etc/setenv && \ + wdkQuery -model $(python3 bin/resolve.py --profile profiles/plasmodb.yml --field model) \ + -query GeneId.GenesByCopyNumber -showQuery"' +``` + +Take the rendered SQL, repoint `apidbtuning.` → `jbrestel.`, and run it read-only in psql. +Repeat for `GenesByCopyNumberComparison` and `SequenceIds.ByCopyNumber`. This verifies the +SQL WDK actually assembles, which the raw model XML does not show. + +After the Jenkins build: run all three live, confirming the organism dropdown is populated +(the old vocab returned zero rows), the samples filter tree renders from EDA, and the +restored CNV columns appear in the results table. + +Note the appDb tunnel target moves between sessions — confirm the database and port before +querying rather than reusing a previous session's. + +## 9. Accepted tradeoffs + +Recorded as decisions with reasons, so they read as intent: + +1. **The shared dnaseq param trio stays in `variationParams.xml`.** The organism → + `eda_sample_table_suffix` → samples-filter chain will serve eight searches across three + record classes (`variation`, `transcript`, `genomic sequence`), and a genomic-sequence + ploidy search referencing a paramSet named `variationParams` is a naming lie. Promoting + it to a neutral file was considered and rejected for this spec: it would mean renaming + `paramRef`s in five searches that currently work, and the naming cost is cosmetic while + the regression risk is not. Revisit if a fourth record class joins. + +2. **The transitional tuning tables create a one-release cleanup.** §7 schedules it + explicitly, because "trivial cleanup next release" is how permanent tables are born. + +3. **Select-all remains slow.** ~4.1 s at 216 samples here; production pfal has ~538 strains + in the HSSS directories, extrapolating to roughly 10 s. The tuning table narrows this but + cannot remove it — the aggregation reads the same row volume either way. Accepted as a + tail case: CNV searches are normally run on a handful of samples. If it becomes a real + complaint, the fix is a materialized per-organism median, not a different table layout. + +4. **`GenesByNgsSnps` gets no `minSelectedCount` change.** It inherits + `variation_sample_meta`'s minimum of 2, which is correct for it — SNP characteristics are + computed from differences *between* selected samples. + +## 10. Out of scope + +- `GeneQuestions.GenesBySnps` (`individuals.txt:116`) — a second orphaned ontology row for a + question the model does not define. Harmless; it belongs to the older, non-HTS snp search + and has no port target in this spec. +- The `FindChipPolymorphismsPlugin` / `FindChipSnpMajorAllelesPlugin` chip searches, still + pointing at `/highSpeedChipSnpSearch`. Equally dead, deliberately untouched. +- Re-running the workflow to populate `webready.*_p`. §7 is precisely the arrangement that + avoids needing it. From 9a31137a8d24c641c675e0d13444d69baffb7381 Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Thu, 6 Aug 2026 12:03:45 -0400 Subject: [PATCH 59/98] Add implementation plan for the Genetic Variation searches port Fifteen tasks in three phases, with a hard halt between phase 2 and 3 for the Jenkins tuning build: phase 3 names apidbtuning tables, and WDK tests sqlQuery definitions during the model build, so it fails outright rather than degrading if they are absent. GenesByNgsSnps is phase 1 and ships alone, since it depends on none of the CNV work. Because three of these four searches currently build fine and silently return nothing, every task establishes a failing observation before editing, rather than treating a green build as evidence. Co-Authored-By: Claude Opus 5 --- ...6-08-06-genetic-variation-searches-port.md | 1694 +++++++++++++++++ 1 file changed, 1694 insertions(+) create mode 100644 docs/superpowers/plans/2026-08-06-genetic-variation-searches-port.md diff --git a/docs/superpowers/plans/2026-08-06-genetic-variation-searches-port.md b/docs/superpowers/plans/2026-08-06-genetic-variation-searches-port.md new file mode 100644 index 0000000000..e94b64cc4f --- /dev/null +++ b/docs/superpowers/plans/2026-08-06-genetic-variation-searches-port.md @@ -0,0 +1,1694 @@ +# Genetic Variation Searches Port — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Port the four remaining Genetic Variation searches — `GenesByNgsSnps`, +`SequencesByPloidy`, `GenesByCopyNumber`, `GenesByCopyNumberComparison` — onto the merged +dnaseq/EDA world, so all eight dnaseq searches work and are categorized together. + +**Architecture:** `GenesByNgsSnps` is uncommented and repointed from the absent `snpParams` +paramSet to its live `variationParams` twins; its HSSS plugin was already fixed by the +2026-08-05 plumbing work. The three CNV searches are rebuilt on corrected copy-number tables +that drop the dead `PANIO_p`/`study.Input` join and key on an EDA sample stable ID. Those +tables land transitionally in `apidbtuning` (buildable now via Jenkins) and permanently in +`webready/*.psql` (next workflow run). + +**Tech Stack:** WDK model XML (questions, queries, params, ontology), EuPathDB tuningManager +XML, PostgreSQL, the `wb` build wrapper, Claude in Chrome for live QA. + +**Spec:** `ApiCommonModel/docs/superpowers/specs/2026-08-06-genetic-variation-searches-port-design.md` + +--- + +## How "TDD" works here + +There is no unit-test harness for WDK model XML. The test-first discipline still applies, +in this form — **every task establishes a failing observation before changing anything**: + +| Instead of | Do this | +|---|---| +| write a failing test | run the build / query / service endpoint and **capture the current failure or absence** | +| watch it fail | confirm the exact error text or the missing item | +| implement | make the edit | +| watch it pass | re-run the *same* command and confirm the failure is gone | + +Never skip the "before" observation. Three of these four searches currently **build fine and +silently return nothing**, which is exactly the failure mode a "did it build?" check misses. + +## Environment + +All commands run from `/home/jbrestel/workspaces/agentic-veupath-dev` unless stated. +Source edits are in `/home/jbrestel/workspaces/plasmodb//`, already on branch +`dnaseq-merge-experiments`. Mutagen carries edits to the remote; **do not** run builds +locally. + +| Thing | Value | +|---|---| +| build | `bin/veup-build.sh plasmodb wb model` / `... wb ontology` | +| logs | `bin/veup-logs.sh plasmodb mark From 2d0a87b900ff1e760efdff4599afb2e3edf880dd Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Thu, 6 Aug 2026 14:05:21 -0400 Subject: [PATCH 68/98] Add CNV organism vocabularies and the cnv_sample_meta filter organismVQ.CNV returns zero rows - no datasource matches %copynumbervariations_%, because CNV now rides the merged isolates/ Dna_Seq datasources. Its replacements key on the corrected tuning tables and return the scientific name, matching organismSinglePick. Two vocabularies rather than one: the gene searches join both CNV tables and the ploidy search only the chromosome one, so a shared vocabulary would either hide or over-offer an organism. --- .../model/questions/params/organismParams.xml | 37 +++++++++++++++++++ .../questions/params/variationParams.xml | 30 +++++++++++++++ 2 files changed, 67 insertions(+) diff --git a/Model/lib/wdk/model/questions/params/organismParams.xml b/Model/lib/wdk/model/questions/params/organismParams.xml index e50e79517a..e89210c523 100644 --- a/Model/lib/wdk/model/questions/params/organismParams.xml +++ b/Model/lib/wdk/model/questions/params/organismParams.xml @@ -551,6 +551,43 @@ + + + + + + + + + + + + + + + + + + diff --git a/Model/lib/wdk/model/questions/params/variationParams.xml b/Model/lib/wdk/model/questions/params/variationParams.xml index 3f9de86881..374f4ed747 100644 --- a/Model/lib/wdk/model/questions/params/variationParams.xml +++ b/Model/lib/wdk/model/questions/params/variationParams.xml @@ -76,6 +76,36 @@ + + + + + + + Choose the resequenced strains or samples to examine. Use the sample characteristics + to narrow the group, or accept all samples for the organism you chose. + + + From 8eca4e946e9674316f5b380b3e28f8e270195ab3 Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Thu, 6 Aug 2026 14:11:53 -0400 Subject: [PATCH 69/98] Port GenesByCopyNumber onto the corrected CNV tables Replaces the input_pan_id/output_pan_id joins, which had no data behind them, with the EDA sample stable id, and scopes on organism rather than the retired organismSinglePickCnv. The organism vocabulary is a queryRef override on organismSinglePick rather than a new param, so eda_sample_table_suffix and cnv_sample_meta stay attached to the organism they are scoped by. Co-Authored-By: Claude Opus 5 --- .../model/questions/queries/geneQueries.xml | 25 ++++++++++++------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/Model/lib/wdk/model/questions/queries/geneQueries.xml b/Model/lib/wdk/model/questions/queries/geneQueries.xml index a7236eb672..5fe0de91ca 100644 --- a/Model/lib/wdk/model/questions/queries/geneQueries.xml +++ b/Model/lib/wdk/model/questions/queries/geneQueries.xml @@ -5496,8 +5496,15 @@ select distinct ta.gene_source_id - - + + + + @@ -5523,19 +5530,19 @@ select distinct ta.gene_source_id SELECT DISTINCT g.project_id , g.source_id , g.gene_source_id - , g.strain + , g.eda_sample_stable_id AS strain , g.raw_estimate , g.ref_cn , g.haploid_number , c.ploidy , g.chromosome - FROM webready.GeneCopyNumbers_p g - , webready.ChrCopyNumbers_p c - WHERE c.output_pan_id IN ($$CNV_strain$$) - AND g.input_pan_id = c.input_pan_id + FROM apidbtuning.GeneCopyNumbers g + , apidbtuning.ChrCopyNumbers c + WHERE c.eda_sample_stable_id IN ($$cnv_sample_meta$$) + AND g.eda_sample_stable_id = c.eda_sample_stable_id AND g.na_sequence_id = c.na_sequence_id - AND g.org_abbrev = $$organismSinglePickCnv$$ - AND c.org_abbrev = $$organismSinglePickCnv$$ + AND g.organism = $$organismSinglePick$$ + AND c.organism = $$organismSinglePick$$ ) , medians AS ( SELECT s.gene_source_id From 9123b6a949f372acb98b7092d95c0861256be95f Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Thu, 6 Aug 2026 14:16:55 -0400 Subject: [PATCH 70/98] Use percentile_cont, not Oracle median(), in the CNV medians CTEs median() is an Oracle aggregate and does not exist in Postgres, so both gene CNV searches would fail at runtime with 'function median(numeric) does not exist' even once their data joins were fixed. Half-finished migration: the hit_medians CTE in each of these same two queries already used percentile_cont(0.5) WITHIN GROUP. This brings the medians CTE alongside it. The two forms are equivalent - both do continuous interpolation - so results are unchanged. compoundQueries.xml has four further median() occurrences belonging to unrelated searches; out of scope here. --- .../model/questions/queries/geneQueries.xml | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/Model/lib/wdk/model/questions/queries/geneQueries.xml b/Model/lib/wdk/model/questions/queries/geneQueries.xml index 5fe0de91ca..b43f88da30 100644 --- a/Model/lib/wdk/model/questions/queries/geneQueries.xml +++ b/Model/lib/wdk/model/questions/queries/geneQueries.xml @@ -5546,10 +5546,12 @@ select distinct ta.gene_source_id ) , medians AS ( SELECT s.gene_source_id - , median (s.ploidy) AS median_ploidy - , median (s.raw_estimate) AS median_raw - , median (s.haploid_number) AS median_haploid - , median (s.ploidy * s.haploid_number) AS median_gene_dose + -- percentile_cont, not Oracle's median(): median() does not exist in + -- Postgres. The sibling hit_medians CTE below already uses this form. + , percentile_cont(0.5) WITHIN GROUP (ORDER BY s.ploidy) AS median_ploidy + , percentile_cont(0.5) WITHIN GROUP (ORDER BY s.raw_estimate) AS median_raw + , percentile_cont(0.5) WITHIN GROUP (ORDER BY s.haploid_number) AS median_haploid + , percentile_cont(0.5) WITHIN GROUP (ORDER BY s.ploidy * s.haploid_number) AS median_gene_dose FROM bySample s GROUP BY s.gene_source_id ) @@ -5642,10 +5644,12 @@ select distinct ta.gene_source_id AND c.org_abbrev = $$organismSinglePickCnv$$ ) , medians AS ( SELECT s.gene_source_id - , median (s.ploidy) AS median_ploidy - , median (s.raw_estimate) AS median_raw - , median (s.haploid_number) AS median_haploid - , median (s.ploidy * s.haploid_number) AS median_gene_dose + -- percentile_cont, not Oracle's median(): median() does not exist in + -- Postgres. The sibling hit_medians CTE below already uses this form. + , percentile_cont(0.5) WITHIN GROUP (ORDER BY s.ploidy) AS median_ploidy + , percentile_cont(0.5) WITHIN GROUP (ORDER BY s.raw_estimate) AS median_raw + , percentile_cont(0.5) WITHIN GROUP (ORDER BY s.haploid_number) AS median_haploid + , percentile_cont(0.5) WITHIN GROUP (ORDER BY s.ploidy * s.haploid_number) AS median_gene_dose FROM bySample s GROUP BY s.gene_source_id ) , hit_medians AS ( From 6d32f8ca48d77804392cc4962450c85c5ee843a1 Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Thu, 6 Aug 2026 14:21:42 -0400 Subject: [PATCH 71/98] Port GenesByCopyNumberComparison onto the corrected CNV tables Same treatment as GenesByCopyNumber: the input_pan_id/output_pan_id joins had no data behind them, so sample identity moves to the EDA sample stable id and scoping to organism rather than the retired organismSinglePickCnv. --- .../model/questions/queries/geneQueries.xml | 25 ++++++++++++------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/Model/lib/wdk/model/questions/queries/geneQueries.xml b/Model/lib/wdk/model/questions/queries/geneQueries.xml index b43f88da30..7aeb289eed 100644 --- a/Model/lib/wdk/model/questions/queries/geneQueries.xml +++ b/Model/lib/wdk/model/questions/queries/geneQueries.xml @@ -5604,8 +5604,15 @@ select distinct ta.gene_source_id - - + + + + @@ -5629,19 +5636,19 @@ select distinct ta.gene_source_id SELECT DISTINCT g.project_id , g.source_id , g.gene_source_id - , g.strain + , g.eda_sample_stable_id AS strain , g.raw_estimate , g.ref_cn , g.haploid_number , c.ploidy , g.chromosome - FROM webready.GeneCopyNumbers_p g - , webready.ChrCopyNumbers_p c - WHERE c.output_pan_id IN ($$CNV_strain$$) - AND g.input_pan_id = c.input_pan_id + FROM apidbtuning.GeneCopyNumbers g + , apidbtuning.ChrCopyNumbers c + WHERE c.eda_sample_stable_id IN ($$cnv_sample_meta$$) + AND g.eda_sample_stable_id = c.eda_sample_stable_id AND g.na_sequence_id = c.na_sequence_id - AND g.org_abbrev = $$organismSinglePickCnv$$ - AND c.org_abbrev = $$organismSinglePickCnv$$ + AND g.organism = $$organismSinglePick$$ + AND c.organism = $$organismSinglePick$$ ) , medians AS ( SELECT s.gene_source_id -- percentile_cont, not Oracle's median(): median() does not exist in From f95b152be15900366187bdab4d1fb1299fde9d98 Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Thu, 6 Aug 2026 14:26:03 -0400 Subject: [PATCH 72/98] Restore the CNV summary columns on both gene CNV searches Both attributesList blocks were commented out, so ten declared CNV columns never reached the results table and the searches returned bare gene rows. Restored with the CNV columns only. --- .../lib/wdk/model/questions/geneQuestions.xml | 27 ++++++++++++------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/Model/lib/wdk/model/questions/geneQuestions.xml b/Model/lib/wdk/model/questions/geneQuestions.xml index 57731cffc2..70402ca6a1 100644 --- a/Model/lib/wdk/model/questions/geneQuestions.xml +++ b/Model/lib/wdk/model/questions/geneQuestions.xml @@ -4289,11 +4289,20 @@ In this study, genome-wide expression level polymorphisms (ELPs) were examined i recordClassRef="TranscriptRecordClasses.TranscriptRecordClass" includeProjects="AmoebaDB,CryptoDB,PlasmoDB,ToxoDB,TriTrypDB,FungiDB,UniDB" newBuild="24"> - ---> + summary="strains,ref_cn,median_raw_hits,median_haploid_hits,median_ploidy_hits,median_gene_dose_hits" + sorting="median_haploid_hits desc"/> Find genes based on the number of copies in resequenced strains - ---> - Find genes by comparing the gene copy number in the resequenced strain to its copy + summary="strains,ref_cn,median_raw_hits,median_haploid_hits,median_ploidy_hits,median_gene_dose_hits" + sorting="median_haploid_hits desc"/> + Find genes by comparing the gene copy number in the resequenced strain to its copy number in the reference genome. Date: Thu, 6 Aug 2026 14:31:08 -0400 Subject: [PATCH 73/98] Port SequencesByPloidy onto the corrected chromosome CNV table Also adds an explicit organism predicate. The query previously had none, relying on CNV_strain's organism-scoped vocabulary; with samples now coming from an EDA study, a name colliding across organisms would leak rows. Retires the strains regex too - it required a space these sample names never contain, so it stripped nothing and displayed 427_Ploidy. --- .../questions/queries/genomicQueries.xml | 29 ++++++++++--------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/Model/lib/wdk/model/questions/queries/genomicQueries.xml b/Model/lib/wdk/model/questions/queries/genomicQueries.xml index dbb1a3031f..e69909a783 100644 --- a/Model/lib/wdk/model/questions/queries/genomicQueries.xml +++ b/Model/lib/wdk/model/questions/queries/genomicQueries.xml @@ -308,8 +308,13 @@ - - + + + + @@ -320,17 +325,13 @@ Date: Thu, 6 Aug 2026 14:40:13 -0400 Subject: [PATCH 74/98] Categorize the three CNV searches under Genetic Variation SequencesByPloidy moves from Curation and Annotation, so all eight dnaseq searches sit in one menu section. --- Model/lib/wdk/ontology/individuals.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Model/lib/wdk/ontology/individuals.txt b/Model/lib/wdk/ontology/individuals.txt index a91186ee6c..738191b61a 100644 --- a/Model/lib/wdk/ontology/individuals.txt +++ b/Model/lib/wdk/ontology/individuals.txt @@ -97,8 +97,8 @@ TranscriptRecordClasses.TranscriptRecordClass.GeneQuestions.GenesByReactionCompo TranscriptRecordClasses.TranscriptRecordClass.GeneQuestions.GenesByMolecularWeight http://edamontology.org/topic_0123 Protein properties TranscriptRecordClasses.TranscriptRecordClass search GeneQuestions.GenesByMolecularWeight menu webservice TranscriptRecordClasses.TranscriptRecordClass.GeneQuestions.GenesByMotifSearch http://edamontology.org/topic_0080 Sequence Analysis TranscriptRecordClasses.TranscriptRecordClass search GeneQuestions.GenesByMotifSearch menu webservice TranscriptRecordClasses.TranscriptRecordClass.GeneQuestions.GenesByNgsSnps http://edamontology.org/topic_0199 Genetic Variation TranscriptRecordClasses.TranscriptRecordClass search GeneQuestions.GenesByNgsSnps menu webservice -##TranscriptRecordClasses.TranscriptRecordClass.GeneQuestions.GenesByCopyNumber http://edamontology.org/topic_0199 Genetic Variation TranscriptRecordClasses.TranscriptRecordClass search GeneQuestions.GenesByCopyNumber menu webservice -##TranscriptRecordClasses.TranscriptRecordClass.GeneQuestions.GenesByCopyNumberComparison http://edamontology.org/topic_0199 Genetic Variation TranscriptRecordClasses.TranscriptRecordClass search GeneQuestions.GenesByCopyNumberComparison menu webservice +TranscriptRecordClasses.TranscriptRecordClass.GeneQuestions.GenesByCopyNumber http://edamontology.org/topic_0199 Genetic Variation TranscriptRecordClasses.TranscriptRecordClass search GeneQuestions.GenesByCopyNumber menu webservice +TranscriptRecordClasses.TranscriptRecordClass.GeneQuestions.GenesByCopyNumberComparison http://edamontology.org/topic_0199 Genetic Variation TranscriptRecordClasses.TranscriptRecordClass search GeneQuestions.GenesByCopyNumberComparison menu webservice TranscriptRecordClasses.TranscriptRecordClass.GeneQuestions.GenesByNonnuclearLocation GenomicSequenceLocationCategory GenomicSequenceLocationCategory TranscriptRecordClasses.TranscriptRecordClass search GeneQuestions.GenesByNonnuclearLocation menu webservice TranscriptRecordClasses.TranscriptRecordClass.GeneQuestions.GenesByOldAnnotation http://edamontology.org/topic_0219 Curation and Annotation TranscriptRecordClasses.TranscriptRecordClass search GeneQuestions.GenesByOldAnnotation TranscriptRecordClasses.TranscriptRecordClass.GeneQuestions.GenesByOrthologPattern http://edamontology.org/topic_3299 Evolutionary Biology TranscriptRecordClasses.TranscriptRecordClass search GeneQuestions.GenesByOrthologPattern menu webservice @@ -135,7 +135,7 @@ SequenceRecordClasses.SequenceRecordClass.GenomicSequenceQuestions.SequenceByWei SequenceRecordClasses.SequenceRecordClass.GenomicSequenceQuestions.SequencesBySimilarity http://edamontology.org/topic_0080 Sequence Analysis SequenceRecordClasses.SequenceRecordClass search GenomicSequenceQuestions.SequencesBySimilarity webservice SequenceRecordClasses.SequenceRecordClass.GenomicSequenceQuestions.SequencesByMultiBlast http://edamontology.org/topic_0080 Sequence Analysis SequenceRecordClasses.SequenceRecordClass search GenomicSequenceQuestions.SequencesByMultiBlast menu webservice SequenceRecordClasses.SequenceRecordClass.GenomicSequenceQuestions.SequencesByTaxon http://edamontology.org/topic_0637 Taxonomy SequenceRecordClasses.SequenceRecordClass search GenomicSequenceQuestions.SequencesByTaxon menu webservice -##SequenceRecordClasses.SequenceRecordClass.GenomicSequenceQuestions.SequencesByPloidy http://edamontology.org/topic_0219 Curation and Annotation SequenceRecordClasses.SequenceRecordClass search GenomicSequenceQuestions.SequencesByPloidy menu webservice +SequenceRecordClasses.SequenceRecordClass.GenomicSequenceQuestions.SequencesByPloidy http://edamontology.org/topic_0199 Genetic Variation SequenceRecordClasses.SequenceRecordClass search GenomicSequenceQuestions.SequencesByPloidy menu webservice TranscriptRecordClasses.TranscriptRecordClass.InternalGeneDatasetQuestions.GenesByProteinArray http://edamontology.org/topic_0804 Immunology TranscriptRecordClasses.TranscriptRecordClass search InternalGeneDatasetQuestions.GenesByProteinArray menu TranscriptRecordClasses.TranscriptRecordClass.InternalGeneDatasetQuestions.GenesByMicroarrayEvidence http://edamontology.org/topic_3308 Transcriptomics TranscriptRecordClasses.TranscriptRecordClass search InternalGeneDatasetQuestions.GenesByMicroarrayEvidence menu TranscriptRecordClasses.TranscriptRecordClass.InternalGeneDatasetQuestions.GenesByQuantitativeProteomics http://edamontology.org/topic_0121 Proteomics TranscriptRecordClasses.TranscriptRecordClass search InternalGeneDatasetQuestions.GenesByQuantitativeProteomics menu From 52a1b82b174d9dd780b6e4af318be76f7a9ab758 Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Thu, 6 Aug 2026 14:45:57 -0400 Subject: [PATCH 75/98] Fix Task 14's unpassable gate in the port plan The gate asked for zero assembled-model references to the params being retired, but wdkXml -model dumps a registry of every declared param and query, so the deletion targets necessarily appear in their own inventory entries. It could never return zero. The real condition is that every reference comes from within the deletion set itself, and that no Question references any of them. Both now stated explicitly, with the verified line-by-line breakdown. Co-Authored-By: Claude Opus 5 --- ...6-08-06-genetic-variation-searches-port.md | 31 +++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/docs/superpowers/plans/2026-08-06-genetic-variation-searches-port.md b/docs/superpowers/plans/2026-08-06-genetic-variation-searches-port.md index e5c75f004d..743da518e9 100644 --- a/docs/superpowers/plans/2026-08-06-genetic-variation-searches-port.md +++ b/docs/superpowers/plans/2026-08-06-genetic-variation-searches-port.md @@ -1640,8 +1640,35 @@ grep -c "organismSinglePickCnv\|CNV_strain\|CnvSamplesMetadataByOrganism\|CnvMet /tmp/claude-1000/-home-jbrestel-workspaces-agentic-veupath-dev/0c03b31d-4bcd-456b-b2ac-4cbb0a3afb4f/scratchpad/assembled.xml ``` -Expected: `0`. **If it is not zero, stop** — something still references them and deleting -would break the model build. +Expected: **4, not 0** — and that is correct, not a problem. + +`wdkXml -model` dumps a full *registry* of every declared param and query, so the items being +deleted necessarily appear in their own inventory entries. A grep-for-zero gate here **cannot +pass by construction**. (This was a defect in an earlier draft of this plan. An agent hit it, +correctly refused to reinterpret the premise on its own authority, and stopped — the right +call, and why the gate is now written properly.) + +**The real gate:** every reference must come from *within the deletion set itself*. Verified +on this instance: + +| line | reference | +|---|---| +| 400 | `organismSinglePickCnv`'s own declaration | +| 1405 | `CNV_strain`'s own declaration | +| 8961 | `organismVQ.CNV`'s own declaration | +| 9121, 9124 | the two doomed `SharedVQ` queries, each naming the doomed param | + +**Stop only if a *Question* references one of them.** Check that specifically: + +```bash +grep -nE "GenesByCopyNumber|GenesByCopyNumberComparison|ByCopyNumber" \ + /assembled.xml | grep -E "organismSinglePickCnv|CNV_strain" +``` + +Expected: no output — by this point `GeneId.GenesByCopyNumber` lists `organismSinglePick` +and `cnv_sample_meta` instead. + +The grep-for-zero check *is* meaningful **after** deletion; that is Step 5's job. - [ ] **Step 2: Delete from `organismParams.xml`** From e0bbc54877763fe33ec1bd8f9d230ab4d35453a0 Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Thu, 6 Aug 2026 14:50:53 -0400 Subject: [PATCH 76/98] Retire the dead CNV organism and strain params organismVQ.CNV returns zero rows; CnvSamplesMetadataByOrganism and CnvMetadataSpecByOrganism read apidbTuning.Metadata and .Ontology, which do not exist in this build. Their last consumers were ported off in the preceding commits. Co-Authored-By: Claude Opus 5 --- .../model/questions/params/organismParams.xml | 46 --------------- .../model/questions/params/sharedParams.xml | 57 ------------------- 2 files changed, 103 deletions(-) diff --git a/Model/lib/wdk/model/questions/params/organismParams.xml b/Model/lib/wdk/model/questions/params/organismParams.xml index e89210c523..992e432bcf 100644 --- a/Model/lib/wdk/model/questions/params/organismParams.xml +++ b/Model/lib/wdk/model/questions/params/organismParams.xml @@ -234,25 +234,6 @@ - - - - Select the organism you wish to query against. - - - - - - - showOnlyPreferredOrganisms - - - - - - - - - SELECT DISTINCT tn.NAME as term - , string_agg(o.abbrev, ',') as internal - FROM APIDB.DATASOURCE d - , SRES.TAXONNAME tn - , SRES.EXTERNALDATABASE ed - , SRES.EXTERNALDATABASERELEASE edr - , STUDY.nodeset s1 - , apidb.organism o - WHERE lower(d.NAME) like '%copynumbervariations_%' - AND tn.TAXON_ID = d.TAXON_ID - AND tn.taxon_id = o.taxon_id - AND tn.NAME_CLASS = 'scientific name' - AND ed.name = d.name - AND edr.VERSION = d.VERSION - AND edr.EXTERNAL_DATABASE_ID = ed.EXTERNAL_DATABASE_ID - AND s1.EXTERNAL_DATABASE_RELEASE_ID = edr.EXTERNAL_DATABASE_RELEASE_ID - GROUP BY tn.name - ORDER BY tn.NAME - - - - - - - - Choose a resequenced strain or sample that has been mapped against the organism's reference genome. Genomic sequences returned by this search will be part of this strain's genome. - - - --> - - - - - - - - - - - - - - - - - - - - - - - - - - From 63d6e802e809ca53e4d980c45e76122c71e733f7 Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Thu, 6 Aug 2026 15:39:59 -0400 Subject: [PATCH 77/98] Compute ref_cn from the annotation, per chromosome apidb.genecopynumber.ref_copy_number is computed per sample: the loader counts only same-ortholog-group genes present in that sample's own result set, so a partially covered sample undercounts it. Verified: of 72 pfal genes in group OG7_0000041, M283 has 46 present and stores 46, G213 51, D003 63, while the 202 fully covered samples all store 72. hit_medians GROUPs BY ref_cn, so those genes emitted several rows and WDK rejected the answer - 'Joined attribute query returned a different number of rows (376) than the ID query alone (100)' - which 500'd both gene CNV searches in the UI. Computing it from the annotation instead makes it constant per gene and matches the column's documented definition, which asks for genes on the same chromosome in the same ortholog group. Verified: fan-out at the 216-sample selection drops from 4 rows per id to 1. The loader remains worth fixing separately. Co-Authored-By: Claude Opus 5 --- .../orgSpecific/GeneCopyNumbers_p.psql | 33 +++++++++++++++---- .../xml/tuningManager/apiTuningManager.xml | 29 ++++++++++++++-- 2 files changed, 53 insertions(+), 9 deletions(-) diff --git a/Model/lib/psql/webready/orgSpecific/GeneCopyNumbers_p.psql b/Model/lib/psql/webready/orgSpecific/GeneCopyNumbers_p.psql index b7eb3063a8..ba151f1234 100644 --- a/Model/lib/psql/webready/orgSpecific/GeneCopyNumbers_p.psql +++ b/Model/lib/psql/webready/orgSpecific/GeneCopyNumbers_p.psql @@ -7,6 +7,23 @@ -- The PANIO_p join was removed: study.Input has no rows, so PANIO_p is empty -- and this table came out empty. Organism identity now comes from -- TranscriptAttributes_p and sample identity from the protocolappnode name. + -- + -- ref_cn is computed from the annotation (same ortholog group, same chromosome) + -- and deliberately IGNORES apidb.genecopynumber.ref_copy_number, which the + -- loader computes per sample and which therefore undercounts for partially + -- covered samples. That made ref_cn vary per gene, and hit_medians GROUPs BY it, + -- so WDK rejected the answer with a row-count mismatch. + WITH grp AS ( + SELECT DISTINCT ta.gene_source_id, ta.chromosome, oga.group_id + FROM :SCHEMA.TranscriptAttributes_p ta + JOIN apidb.orthologgroupaasequence oga ON oga.aa_sequence_id = ta.aa_sequence_id + WHERE ta.org_abbrev = ':ORG_ABBREV' + ), refcn AS ( + SELECT a.gene_source_id, count(DISTINCT b.gene_source_id) AS ref_cn + FROM grp a + JOIN grp b ON b.group_id = a.group_id AND b.chromosome = a.chromosome + GROUP BY 1 + ) SELECT DISTINCT ':PROJECT_ID' as project_id , ':ORG_ABBREV' as org_abbrev , ta.organism @@ -16,19 +33,21 @@ , ta.gene_source_id , regexp_replace(pan.name, '_GeneCNV$', '') AS eda_sample_stable_id , gcn.haploid_number AS raw_estimate - , gcn.ref_copy_number AS ref_cn + , COALESCE(r.ref_cn, 1) AS ref_cn , CASE WHEN (gcn.haploid_number < 0.01) THEN 0 WHEN (0.01 < gcn.haploid_number AND gcn.haploid_number < 1.85) THEN 1 ELSE round(gcn.haploid_number) END AS haploid_number , ta.chromosome , ta.na_sequence_id FROM apidb.genecopynumber gcn - , study.protocolappnode pan - , :SCHEMA.TranscriptAttributes_p ta - WHERE gcn.protocol_app_node_id = pan.protocol_app_node_id - AND gcn.na_feature_id = ta.gene_na_feature_id - AND (ta.gene_type = 'protein coding' or ta.gene_type = 'protein coding gene') - AND ta.org_abbrev = ':ORG_ABBREV'; + JOIN study.protocolappnode pan + ON pan.protocol_app_node_id = gcn.protocol_app_node_id + JOIN :SCHEMA.TranscriptAttributes_p ta + ON ta.gene_na_feature_id = gcn.na_feature_id + LEFT JOIN refcn r + ON r.gene_source_id = ta.gene_source_id + WHERE ta.gene_type IN ('protein coding', 'protein coding gene') + AND ta.org_abbrev = ':ORG_ABBREV'; :DECLARE_PARTITION; diff --git a/Model/lib/xml/tuningManager/apiTuningManager.xml b/Model/lib/xml/tuningManager/apiTuningManager.xml index 7801f32407..e439ffb457 100644 --- a/Model/lib/xml/tuningManager/apiTuningManager.xml +++ b/Model/lib/xml/tuningManager/apiTuningManager.xml @@ -3486,7 +3486,17 @@ create index Organism_projectId_idx&1 ON OrganismAttributes&1 (project_id, sourc corrected in the same change but does not take effect until the next workflow run. Scheduled for deletion one release later: see docs/superpowers/specs/2026-08-06-genetic-variation-searches-port-design.md s7. - Any edit here MUST be mirrored in that .psql file, and vice versa. --> + Any edit here MUST be mirrored in that .psql file, and vice versa. + + ref_cn is computed here from the annotation - the count of same-ortholog-group genes + on the same chromosome - and deliberately IGNORES apidb.genecopynumber.ref_copy_number. + That column is contaminated: the loader counts only group members present in each + sample's own result set, so a partially-covered sample undercounts it (verified: of 72 + pfal genes in group OG7_0000041, sample M283 has 46 present and stores 46). Because + hit_medians GROUPs BY ref_cn, that produced multiple rows per gene and WDK rejected the + answer with "Joined attribute query returned a different number of rows". Computing it + here makes it constant per gene AND matches the column's documented definition. The + loader remains worth fixing separately. --> @@ -3494,6 +3504,18 @@ create index Organism_projectId_idx&1 ON OrganismAttributes&1 (project_id, sourc From b13b371b704b69c087f6498cd7d4b3d6e02ada54 Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Thu, 6 Aug 2026 16:48:32 -0400 Subject: [PATCH 78/98] Apply final code review findings Four issues, all confirmed against the live instance before fixing: 1. GenesByNgsSnps had no organism vocabulary override, and the comment claiming the working variation searches use none was false - all four override queryRef to organismVQ.withVariationsTree (variationQueries.xml:44,87,127,170). Without it the dropdown offered every annotated organism, and picking one with no dnaseq study makes eda_sample_table_suffix return zero rows, so the samples filter has nothing to render and the search breaks rather than returning empty. 2. organismVQ.CNVGene and CNVChr had no project predicate, unlike 29 of the file's other vocabularies. The CNV tuning tables span FungiDB, PlasmoDB and TriTrypDB, so a PlasmoDB deployment offered A. fumigatus and would have returned FungiDB transcripts into a PlasmoDB result. Verified after the fix: the vocabulary is now P. falciparum alone, and UniDB still sees all three. 3. The ref_cn comment framed a definitional change as mere decontamination. It is both: the stored column is per-sample AND counts the ortholog group genome-wide, while this counts it per chromosome per the attribute's documented definition. Comments and design s3.5 now say so, including that result sets move. 4. apidb.OrthologGroupAaSequence was an undeclared dependency of the GeneCopyNumbers tuning table, so an ortholog reload would not have triggered a rebuild. Declared. Also documented that ref_cn=1 is the normal case for 89% of pfal genes, so a table built before the orthomcl load would look correct for most rows. Design doc updated for the two-vocabulary split, the ref_cn definition, and a sunset list that was short by two. Co-Authored-By: Claude Opus 5 --- .../orgSpecific/GeneCopyNumbers_p.psql | 6 ++ .../model/questions/params/organismParams.xml | 21 +++++-- .../model/questions/queries/geneQueries.xml | 17 ++++-- .../xml/tuningManager/apiTuningManager.xml | 39 ++++++++++--- ...-genetic-variation-searches-port-design.md | 58 +++++++++++++++++-- 5 files changed, 119 insertions(+), 22 deletions(-) diff --git a/Model/lib/psql/webready/orgSpecific/GeneCopyNumbers_p.psql b/Model/lib/psql/webready/orgSpecific/GeneCopyNumbers_p.psql index ba151f1234..80bbbebe84 100644 --- a/Model/lib/psql/webready/orgSpecific/GeneCopyNumbers_p.psql +++ b/Model/lib/psql/webready/orgSpecific/GeneCopyNumbers_p.psql @@ -33,6 +33,12 @@ , ta.gene_source_id , regexp_replace(pan.name, '_GeneCNV$', '') AS eda_sample_stable_id , gcn.haploid_number AS raw_estimate + -- 1 = the gene itself, for a gene in no ortholog group (67 such in pfal). + -- CAUTION: ref_cn=1 is also the NORMAL result - 89% of pfal genes have no + -- same-chromosome paralog - so if this table is built before the orthomcl + -- load has run, it is silently indistinguishable from a correct one for most + -- rows. The tuningManager copy guards this with an externalDependency on + -- apidb.OrthologGroupAaSequence; here it is a workflow ORDERING requirement. , COALESCE(r.ref_cn, 1) AS ref_cn , CASE WHEN (gcn.haploid_number < 0.01) THEN 0 WHEN (0.01 < gcn.haploid_number AND gcn.haploid_number < 1.85) THEN 1 diff --git a/Model/lib/wdk/model/questions/params/organismParams.xml b/Model/lib/wdk/model/questions/params/organismParams.xml index 992e432bcf..3c48dfc26d 100644 --- a/Model/lib/wdk/model/questions/params/organismParams.xml +++ b/Model/lib/wdk/model/questions/params/organismParams.xml @@ -509,9 +509,18 @@ GenesByCopyNumberComparison, which join both CNV tables, so an organism must appear in the gene table to be useful to them. - internal is the SCIENTIFIC NAME, matching organismSinglePick's convention. The - organismVQ.CNV query this replaces returned string_agg(o.abbrev) - a comma-joined - list of abbreviations - which is a second reason nothing downstream lined up. --> + term is the SCIENTIFIC NAME, which is the value that actually matters: consumers + set noTranslation="true", so $$organismSinglePick$$ interpolates the term, and both + VariationVQ.EdaSampleTableSuffix (tn.name = ...) and these tables' organism column + hold that name. internal is set to the same string only so the two columns agree; + nothing reads it. The organismVQ.CNV query this replaces returned + string_agg(o.abbrev) as internal AND had no usable term, which is part of why + nothing downstream lined up. + + The project predicate is the file's standard idiom and is load-bearing here: these + tuning tables span every project that has CNV data (FungiDB, PlasmoDB, TriTrypDB + today), so without it a PlasmoDB deployment would offer A. fumigatus and return + FungiDB transcripts into a PlasmoDB result. --> @@ -520,6 +529,7 @@ @@ -528,7 +538,9 @@ + would otherwise vanish from this search. + + Same project predicate, for the same reason as CNVGene above. --> @@ -537,6 +549,7 @@ diff --git a/Model/lib/wdk/model/questions/queries/geneQueries.xml b/Model/lib/wdk/model/questions/queries/geneQueries.xml index 7aeb289eed..dced67a7c1 100644 --- a/Model/lib/wdk/model/questions/queries/geneQueries.xml +++ b/Model/lib/wdk/model/questions/queries/geneQueries.xml @@ -2842,10 +2842,19 @@ ]]> - - + + The organism you choose will determine the samples from which you can identify SNPs. diff --git a/Model/lib/xml/tuningManager/apiTuningManager.xml b/Model/lib/xml/tuningManager/apiTuningManager.xml index e439ffb457..82c29f7271 100644 --- a/Model/lib/xml/tuningManager/apiTuningManager.xml +++ b/Model/lib/xml/tuningManager/apiTuningManager.xml @@ -3488,19 +3488,34 @@ create index Organism_projectId_idx&1 ON OrganismAttributes&1 (project_id, sourc docs/superpowers/specs/2026-08-06-genetic-variation-searches-port-design.md s7. Any edit here MUST be mirrored in that .psql file, and vice versa. - ref_cn is computed here from the annotation - the count of same-ortholog-group genes - on the same chromosome - and deliberately IGNORES apidb.genecopynumber.ref_copy_number. - That column is contaminated: the loader counts only group members present in each - sample's own result set, so a partially-covered sample undercounts it (verified: of 72 - pfal genes in group OG7_0000041, sample M283 has 46 present and stores 46). Because - hit_medians GROUPs BY ref_cn, that produced multiple rows per gene and WDK rejected the - answer with "Joined attribute query returned a different number of rows". Computing it - here makes it constant per gene AND matches the column's documented definition. The - loader remains worth fixing separately. --> + ref_cn is computed here from the annotation and deliberately IGNORES + apidb.genecopynumber.ref_copy_number. This is a DEFINITIONAL CHANGE, not merely a + cleanup - read both parts: + + 1. The stored column is per-sample. The loader counts only ortholog-group members + present in that sample's own result set, so a partially-covered sample reports + less (verified: of 72 pfal genes in group OG7_0000041, sample M283 has 46 present + and stores 46, while the 202 fully-covered samples all store 72). Because + hit_medians GROUPs BY ref_cn, that emitted several rows per gene and WDK rejected + the answer - "Joined attribute query returned a different number of rows". + + 2. The stored column also counts the group GENOME-WIDE, whereas the attribute's own + help text says "both on the same chromosome and in the same ortholog group" + (geneQuestions.xml). This computes the same-chromosome count, which matches the + documentation but is SMALLER for any group spread across chromosomes - OG7_0000041's + 72 pfal genes span 13 chromosomes, so its members go from 72 to between 1 and 12. + Roughly a quarter to a half of all rows change value, and + GenesByCopyNumberComparison's entire predicate is haploid_number ref_cn, so + result sets move. Approved deliberately; see the design doc s3.1. + + The loader remains worth fixing separately. --> + + ref_cn`. Verified impact on a 10-sample pfal run: amplified calls move +3,411 → 3,522 (+3%), because 3,933 of 5,285 genes are unchanged either way. + +`COALESCE(…, 1)` covers a gene in no ortholog group — 67 in pfal. Note `ref_cn = 1` is also +the *normal* result (89% of pfal genes have no same-chromosome paralog), so a table built +before the orthomcl load would be silently indistinguishable from a correct one for most +rows. The tuningManager copy guards this with an `externalDependency` on +`apidb.OrthologGroupAaSequence`; in the webready copy it is a workflow ordering requirement. + +**The loader is still wrong** and worth a separate ticket — every rebuild keeps writing +sample-contaminated values into `apidb.genecopynumber` for anyone else reading it. + ## 4. Params ### 4.1 CNV organism: a `queryRef` override, not a new param @@ -192,7 +233,7 @@ interpolate `$$organismSinglePick$$`. A genuinely new organism param would leave pointing at a param the query no longer has, so the samples filter would silently never scope to the chosen organism. -`organismVQ.CNVDnaSeq` returns `internal` = **scientific name**, matching +`organismVQ.CNVGene` / `organismVQ.CNVChr` (two, see below) return `term` = **scientific name**, matching `organismSinglePick`'s convention. The dead `organismVQ.CNV` returned `string_agg(o.abbrev)` — a comma-joined list of abbreviations — which is a second reason nothing downstream lined up. It lists organisms that actually have CNV rows, so an organism @@ -251,7 +292,7 @@ spec, §4.4): | `snpParams.MinPercentMinorAlleles` | `variationParams.MinPercentMinorAlleles` | | `snpParams.MinPercentIsolateCalls` | `variationParams.MinPercentIsolateCalls` | | `snpParams.ngsSnp_strain_meta` | `variationParams.variation_sample_meta` | -| `organismSinglePick` + `queryRef="organismVQ.withNgsSNPsTree"` | plain `organismSinglePick` (drop the override) | +| `organismSinglePick` + `queryRef="organismVQ.withNgsSNPsTree"` | `organismSinglePick` + `queryRef="organismVQ.withVariationsTree"` (swap the override, do **not** drop it) | The strain-filter repoint is not a preference: `FindGenesWithSnpCharsPlugin extends FindPolymorphismsPlugin`, so it inherits `getStrainFilterParamName()` returning @@ -259,7 +300,14 @@ FindPolymorphismsPlugin`, so it inherits `getStrainFilterParamName()` returning run time, not build time. `organismVQ.withNgsSNPsTree` reads `apidbtuning.snpstrains`, which does not exist in this -build; plain `organismSinglePick` is what the five working variation searches use. +build. The replacement is `organismVQ.withVariationsTree` — what the four working variation +`processQuery`s use (`variationQueries.xml:44, 87, 127, 170`). + +**Do not drop the override entirely.** Plain `organismSinglePick` falls back to +`organismVQ.withGenes`, i.e. every annotated organism in the project. Choosing one with no +dnaseq study makes `eda_sample_table_suffix` return zero rows, so the samples filter has +nothing to render and the search breaks rather than returning an empty result. +`withVariationsTree` restricts to organisms having an `isolates`/`Dna_Seq` datasource. ## 5. Queries @@ -387,7 +435,7 @@ copies of a verified definition rather than two guesses. **Sunset, one release later** — this is a scheduled deletion, not an aspiration: - delete the two `` entries from `apiTuningManager.xml`; -- repoint three queries (§5.2 ×2, §5.3) from `apidbtuning.X` to `webready.X_p`; +- repoint **five** references from `apidbtuning.X` to `webready.X_p`: the three queries (§5.2 ×2, §5.3) **and** the two organism vocabularies `organismVQ.CNVGene` / `organismVQ.CNVChr`, which also name the transitional tables; - change the organism predicate from `organism = …` back to `org_abbrev = …` **only if** partition pruning measures better than the composite index — the `organism` column is carried on the webready table too (§3.3), so no schema change is required either way. From d48ac7de3eea8ec3fd424813ebf5b1ef4710c513 Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Fri, 7 Aug 2026 11:26:34 -0400 Subject: [PATCH 79/98] Add GeneVariationSummary tuning + webready tables Replaces the six SNP attributes retired from the gene record (total_hts_snps, hts_nonsynonymous_snps, hts_synonymous_snps, hts_noncoding_snps, hts_stop_codon_snps, hts_nonsyn_syn_ratio), which are already commented out on master along with the TranscriptAttributes_p psql that fed them. The old set had two defects not carried forward: "synonymous" was a residual (total - nonsyn - stop - noncoding), so anything unclassified inflated it; and the ratio reported 0 when there were no synonymous sites, displaying maximal nonsynonymous signal as the minimum value. One row per gene per project. Effect classes partition total_variants (verified 0 violations), as does the impact summary. Adds what the SNP-only model could not express: indel and MIXED loci (14% of pfal), impact summary, segregating loss-of-function with the common subset separated, site-normalized nucleotide diversity, and sample-size context. Three deliberate design points, detailed in the file comments and the spec: - Two grains. Display counts are per gene, unioned across transcripts, most-severe-wins; pi statistics are confined to the representative longest-CDS transcript, whose id is stored. piN/piS is defined for one CDS. - Suppress, never degrade. Frequency-derived statistics accumulate only over loci clearing an allele floor (pi >= 4, common >= 20, rare >= 100) and are NULL below it, never 0. Sample size is per-locus: min(called_strain_count) is 1 in every loaded organism. Counts remain valid at any sample size, so a single-sample organism keeps everything the old section had. - Ploidy is derived, never hardcoded. Measured pfal 1.01, tbru 2.27, afum 2.01 - afum being a haploid fungus called as diploid, which a hardcoded lookup would encode as truth. Site counts are Nei-Gojobori, derived inline from the genetic code. The pooled synonymous-site fraction in pfal is 17.49%, not the textbook ~25%; without this normalization the median piN/piS is 2.0 (implying genome-wide positive selection), with it 0.512 (the expected purifying-selection signature). pfal median pi of 1.5e-3 per site matches published estimates, and known genes rank as expected - AMA1, MSP1, MSP2 high and common-missense dominated; Kelch13 low and rare dominated. Verified against unidb_shu_a: the tuning SQL builds 27,297 rows matching the validated prototype on all counts, bins, ratios and ploidy; the webready copy builds 5,579 pfal rows matching the tuning copy at zero differences across all compared columns. webready is partitioned LIST (org_abbrev) like its peers. Co-Authored-By: Claude Opus 5 --- .../orgSpecific/GeneVariationSummary_p.psql | 369 ++++++++++++++++++ .../GeneVariationSummary_p_ix.psql | 3 + .../xml/tuningManager/apiTuningManager.xml | 359 +++++++++++++++++ ...026-08-07-gene-variation-summary-design.md | 307 +++++++++++++++ 4 files changed, 1038 insertions(+) create mode 100644 Model/lib/psql/webready/orgSpecific/GeneVariationSummary_p.psql create mode 100644 Model/lib/psql/webready/orgSpecific/GeneVariationSummary_p_ix.psql create mode 100644 docs/superpowers/specs/2026-08-07-gene-variation-summary-design.md diff --git a/Model/lib/psql/webready/orgSpecific/GeneVariationSummary_p.psql b/Model/lib/psql/webready/orgSpecific/GeneVariationSummary_p.psql new file mode 100644 index 0000000000..7035d4e214 --- /dev/null +++ b/Model/lib/psql/webready/orgSpecific/GeneVariationSummary_p.psql @@ -0,0 +1,369 @@ +:CREATE_AND_POPULATE + -- Mirrors the GeneVariationSummary tuningTable in apiTuningManager.xml, which + -- carries the gene-record attributes until this file's next workflow run. Any + -- edit here MUST be mirrored there. Full design, validation evidence, and the + -- biologist-facing help text: + -- docs/superpowers/specs/2026-08-07-gene-variation-summary-design.md + -- + -- Replaces six attributes retired from geneRecord.xml (total_hts_snps, + -- hts_nonsynonymous_snps, hts_synonymous_snps, hts_noncoding_snps, + -- hts_stop_codon_snps, hts_nonsyn_syn_ratio), already commented out on master + -- along with the TranscriptAttributes_p psql that fed them. The old + -- "synonymous" was a RESIDUAL (total - nonsyn - stop - noncoding), so anything + -- the old pipeline failed to classify inflated it; the retired values are not + -- a target to reproduce. + -- + -- THIS FILE IS ORG-SPECIFIC. Everything below is scoped to :ORG_ABBREV, so the + -- effective ploidy, allele counts, and site fractions are all per-organism - + -- which is correct, because they genuinely differ (pfal 1.01, tbru 2.27, + -- afum 2.01). + -- + -- THREE THINGS THAT LOOK LIKE STYLE BUT ARE LOAD-BEARING: + -- + -- 1. TWO GRAINS. Display counts are per GENE (unioned across transcripts, + -- most-severe-wins). The pi statistics are confined to the REPRESENTATIVE + -- longest-CDS transcript, stored as rep_transcript_source_id. piN/piS is + -- defined for one CDS; counting variants across all transcripts while + -- normalizing by one transcript's site counts would let a variant in a + -- transcript-specific exon enter the numerator while that exon's sites + -- never enter the denominator. + -- + -- 2. SUPPRESS, NEVER DEGRADE. Frequency-derived statistics accumulate only + -- over loci clearing an allele floor (pi >= 4, common >= 20, rare >= 100) + -- and are NULL below it, never 0. Sample size is a PER-LOCUS property: + -- min(called_strain_count) is 1 in every loaded organism and 35% of pfal + -- loci have <100 alleles. At the 2-allele floor (one sample plus a + -- reference) MAF can only be 0.5, so pi degenerates to a rescaled variant + -- density. Rendering 0.00 there would reproduce the exact defect of the + -- retired hts_nonsyn_syn_ratio, which reported 0 for genes with no + -- synonymous sites and so displayed maximal signal as minimal. The + -- n_loci_* columns publish how many loci contributed. + -- + -- 3. PLOIDY IS DERIVED, NEVER HARDCODED (total_ploidy_count / + -- called_strain_count). afum measures 2.01 despite being a HAPLOID fungus + -- called as diploid; a hardcoded lookup would encode that calling bug as + -- truth, a measured value self-corrects when it is fixed. + -- + -- MAF is a true ALLELE frequency (verified to within 2e-5 against pfal using + -- total_ploidy_count as denominator), so 2p(1-p) holds at any ploidy. But + -- *_minor_allele_strain_count is a STRAIN count - different units - so + -- "singleton" is defined on allele copies, round(maf * total_ploidy_count) = 1. + -- + -- Source is apidb.VariationEffect ALONE: at gene x locus grain + -- VariationTranscriptProduct contributes 0 pairs it lacks, so the defensive + -- UNION in VariationAttributes is dead code at this grain. + -- + -- Site counts are Nei-Gojobori (1986), derived inline from the genetic code + -- rather than hardcoded, so the code table is the single source of truth. The + -- pooled synonymous-site fraction in pfal is 17.49%, NOT the textbook ~25% - + -- an AT-bias effect worth 1.43x on every gene. Without this normalization the + -- median piN/piS is 2.0 (implying genome-wide positive selection); with it, + -- 0.512 (the expected purifying-selection signature). + -- + -- COST NOTE: the cds_codons CTE expands every CDS into one row per codon. This + -- is the expensive step. Unavoidable for per-gene codon composition. + WITH code(codon, aa) AS (VALUES + ('TTT','F'),('TTC','F'),('TTA','L'),('TTG','L'), + ('CTT','L'),('CTC','L'),('CTA','L'),('CTG','L'), + ('ATT','I'),('ATC','I'),('ATA','I'),('ATG','M'), + ('GTT','V'),('GTC','V'),('GTA','V'),('GTG','V'), + ('TCT','S'),('TCC','S'),('TCA','S'),('TCG','S'), + ('CCT','P'),('CCC','P'),('CCA','P'),('CCG','P'), + ('ACT','T'),('ACC','T'),('ACA','T'),('ACG','T'), + ('GCT','A'),('GCC','A'),('GCA','A'),('GCG','A'), + ('TAT','Y'),('TAC','Y'),('TAA','*'),('TAG','*'), + ('CAT','H'),('CAC','H'),('CAA','Q'),('CAG','Q'), + ('AAT','N'),('AAC','N'),('AAA','K'),('AAG','K'), + ('GAT','D'),('GAC','D'),('GAA','E'),('GAG','E'), + ('TGT','C'),('TGC','C'),('TGA','*'),('TGG','W'), + ('CGT','R'),('CGC','R'),('CGA','R'),('CGG','R'), + ('AGT','S'),('AGC','S'),('AGA','R'),('AGG','R'), + ('GGT','G'),('GGC','G'),('GGA','G'),('GGG','G') + ), + -- Nei-Gojobori: per codon position, the fraction of the 3 possible single-base + -- changes that are synonymous. Stop codons excluded; a change creating a stop + -- is nonsynonymous (falls out, since the amino acid differs). Validated + -- against known degeneracy: ATG and TGG give 0 synonymous sites, four-fold + -- codons exactly 1.0, two-fold 1/3, TTA 0.667 (via TTA<->CTA), CGA 1.333. + nbr AS ( + SELECT c.codon, c.aa, p.pos, b.base, + overlay(c.codon placing b.base from p.pos for 1) AS mutated + FROM code c + CROSS JOIN generate_series(1,3) AS p(pos) + CROSS JOIN (VALUES ('A'),('C'),('G'),('T')) AS b(base) + WHERE c.aa <> '*' AND substr(c.codon, p.pos, 1) <> b.base + ), + posf AS ( + SELECT n.codon, n.pos, count(*) FILTER (WHERE m.aa = n.aa)::numeric / 3 AS f + FROM nbr n JOIN code m ON m.codon = n.mutated + GROUP BY 1,2 + ), + codon_sites AS ( + SELECT codon, sum(f) AS syn_sites, 3 - sum(f) AS nonsyn_sites + FROM posf GROUP BY codon + ), + rep AS ( + SELECT DISTINCT ON (gene_source_id) + gene_source_id, organism, taxon_id, chromosome, gene_na_feature_id, + transcript_source_id, na_feature_id AS rep_na_feature_id, + (gene_end_max - gene_start_min + 1) AS gene_length, cds_length + FROM :SCHEMA.TranscriptAttributes_p + WHERE org_abbrev = ':ORG_ABBREV' + ORDER BY gene_source_id, cds_length DESC NULLS LAST, transcript_source_id + ), + -- LATERAL, not a flat expansion: postgres streams each CDS's codons through + -- the aggregate instead of materializing one row per codon. Measured over all + -- 63,082 genes: 31s vs 52s flat, identical results, much smaller peak + -- footprint. A PL/pgSQL loop would do the same but forces procedural code into + -- both this file and the tuningManager mirror; declarative keeps them + -- comparable. Do not "optimize" into a flat expansion - that was 40% slower. + -- + -- The length filter guards against out-of-frame CDS rows, which would yield + -- garbage codons (477 of 63,765 genome-wide; none in the loaded organisms). + gene_sites AS ( + SELECT r.gene_source_id, s.syn_sites, s.nonsyn_sites + FROM rep r + JOIN :SCHEMA.CodingSequence_p cs ON cs.source_id = r.transcript_source_id + CROSS JOIN LATERAL ( + SELECT sum(k.syn_sites) AS syn_sites, + sum(k.nonsyn_sites) AS nonsyn_sites + FROM regexp_matches(upper(cs.sequence), '.{3}', 'g') AS m(arr) + JOIN codon_sites k ON k.codon = m.arr[1] + ) s + WHERE cs.org_abbrev = ':ORG_ABBREV' + AND length(cs.sequence) % 3 = 0 AND length(cs.sequence) >= 6 + ), + sev AS ( + SELECT e.source, t.gene_source_id, e.na_feature_id, + e.sequence_source_id, e.location, + CASE e.effect + WHEN 'frameshift_variant' THEN 1 + WHEN 'stop_gained' THEN 2 + WHEN 'stop_lost' THEN 2 + WHEN 'start_lost' THEN 2 + WHEN 'splice_acceptor_variant' THEN 3 + WHEN 'splice_donor_variant' THEN 3 + WHEN 'conservative_inframe_deletion' THEN 4 + WHEN 'disruptive_inframe_deletion' THEN 4 + WHEN 'conservative_inframe_insertion' THEN 4 + WHEN 'disruptive_inframe_insertion' THEN 4 + WHEN 'inframe_deletion_unnormalized' THEN 4 + WHEN 'inframe_insertion_unnormalized' THEN 4 + WHEN 'missense_variant' THEN 5 + WHEN 'splice_region_variant' THEN 6 + WHEN 'synonymous_variant' THEN 7 + WHEN 'stop_retained_variant' THEN 7 + WHEN 'start_retained_variant' THEN 7 + WHEN '5_prime_UTR_variant' THEN 8 + WHEN '3_prime_UTR_variant' THEN 8 + WHEN '5_prime_UTR_premature_start_codon_gain_variant' THEN 8 + WHEN 'non_coding_transcript_exon_variant' THEN 9 + WHEN 'non_coding_transcript_variant' THEN 9 + WHEN 'intron_variant' THEN 10 + ELSE 11 + END AS sev, + CASE e.impact WHEN 'HIGH' THEN 4 WHEN 'MODERATE' THEN 3 + WHEN 'LOW' THEN 2 WHEN 'MODIFIER' THEN 1 END AS imp + FROM apidb.VariationEffect e + JOIN :SCHEMA.TranscriptAttributes_p t ON t.na_feature_id = e.na_feature_id + WHERE t.org_abbrev = ':ORG_ABBREV' + ), + vf AS ( + SELECT sequence_source_id, location, variant_type, is_coding, call_rate, + called_strain_count, het_strain_count, indel_frame_effect, + total_ploidy_count AS n_alleles, + nullif(greatest(coalesce(snp_minor_allele_frequency,0), + coalesce(indel_minor_allele_frequency,0)),0) AS maf + FROM apidb.VariationFeature + ), + gene_locus AS ( + SELECT source, gene_source_id, sequence_source_id, location, + min(sev) AS sev, max(imp) AS imp + FROM sev GROUP BY 1,2,3,4 + ), + gl AS ( + SELECT g.*, v.variant_type, v.is_coding, v.call_rate, v.called_strain_count, + v.het_strain_count, v.indel_frame_effect, v.n_alleles, v.maf, + round(v.maf * v.n_alleles) AS minor_copies + FROM gene_locus g JOIN vf v + ON v.sequence_source_id = g.sequence_source_id AND v.location = g.location + ), + snp_agg AS ( + SELECT gene_source_id, + count(*) AS total_variants, + count(*) FILTER (WHERE is_coding=1) AS n_coding_loci, + count(*) FILTER (WHERE variant_type='SNV') AS n_snv, + count(*) FILTER (WHERE variant_type='INDEL') AS n_indel, + count(*) FILTER (WHERE variant_type='MIXED') AS n_mixed, + count(*) FILTER (WHERE sev=1) AS n_frameshift, + count(*) FILTER (WHERE sev=2) AS n_nonsense, + count(*) FILTER (WHERE sev=3) AS n_splice_disruptive, + count(*) FILTER (WHERE sev=4) AS n_inframe_indel, + count(*) FILTER (WHERE sev=5) AS n_missense, + count(*) FILTER (WHERE sev=6) AS n_splice_region, + count(*) FILTER (WHERE sev=7) AS n_synonymous, + count(*) FILTER (WHERE sev=8) AS n_utr, + count(*) FILTER (WHERE sev=9) AS n_noncoding_exon, + count(*) FILTER (WHERE sev=10) AS n_intron, + count(*) FILTER (WHERE sev=11) AS n_other, + count(*) FILTER (WHERE sev<=3) AS n_lof, + count(*) FILTER (WHERE indel_frame_effect='frameshift') AS n_indel_frameshift, + count(*) FILTER (WHERE imp=4) AS n_impact_high, + count(*) FILTER (WHERE imp=3) AS n_impact_moderate, + count(*) FILTER (WHERE imp=2) AS n_impact_low, + count(*) FILTER (WHERE imp=1) AS n_impact_modifier, + CASE max(imp) WHEN 4 THEN 'HIGH' WHEN 3 THEN 'MODERATE' + WHEN 2 THEN 'LOW' WHEN 1 THEN 'MODIFIER' END AS most_severe_impact, + max(called_strain_count) AS max_called_strain_count, + round(percentile_cont(0.5) WITHIN GROUP (ORDER BY called_strain_count)::numeric,0) + AS median_called_strain_count, + max(n_alleles) AS max_alleles, + round(avg(n_alleles/nullif(called_strain_count,0))::numeric,2) AS effective_ploidy, + round(avg(call_rate)::numeric,3) AS avg_call_rate, + round(min(call_rate)::numeric,3) AS min_call_rate, + count(*) FILTER (WHERE call_rate < 0.5) AS n_low_call_rate, + count(*) FILTER (WHERE het_strain_count > 0) AS n_het_loci, + count(*) FILTER (WHERE n_alleles >= 4) AS n_loci_pi, + count(*) FILTER (WHERE n_alleles >= 20) AS n_loci_freq20, + count(*) FILTER (WHERE n_alleles >= 100) AS n_loci_freq100, + count(*) FILTER (WHERE n_alleles >= 20 AND maf > 0.05) AS n_common_raw, + count(*) FILTER (WHERE n_alleles >= 20 AND maf > 0.20) AS n_very_common_raw, + count(*) FILTER (WHERE n_alleles >= 20 AND maf > 0.05 AND sev=5) AS n_missense_common_raw, + count(*) FILTER (WHERE n_alleles >= 20 AND maf > 0.05 AND sev=7) AS n_synonymous_common_raw, + count(*) FILTER (WHERE n_alleles >= 20 AND maf > 0.05 AND sev<=3) AS n_lof_common_raw, + max(maf) FILTER (WHERE n_alleles >= 20) AS max_maf_raw, + count(*) FILTER (WHERE n_alleles >= 100 AND maf <= 0.01) AS n_rare_raw, + count(*) FILTER (WHERE n_alleles >= 100 AND minor_copies = 1) AS n_singleton_raw + FROM gl WHERE source='snpeff' GROUP BY 1 + ), + pc_agg AS ( + SELECT gene_source_id, + count(*) AS pc_total_coding_variants, + count(*) FILTER (WHERE sev=5) AS pc_n_missense, + count(*) FILTER (WHERE sev=7) AS pc_n_synonymous, + count(*) FILTER (WHERE sev<=3) AS pc_n_lof, + count(*) FILTER (WHERE sev=11) AS pc_n_unclassified, + count(*) FILTER (WHERE n_alleles >= 20 AND maf > 0.05 AND sev=5) AS pc_n_missense_common_raw, + count(*) FILTER (WHERE n_alleles >= 20 AND maf > 0.05 AND sev<=3) AS pc_n_lof_common_raw + FROM gl WHERE source='product_call' GROUP BY 1 + ), + tx_locus AS ( + SELECT s.source, r.gene_source_id, s.sequence_source_id, s.location, + min(s.sev) AS sev + FROM sev s JOIN rep r ON r.rep_na_feature_id = s.na_feature_id + GROUP BY 1,2,3,4 + ), + -- unbiased heterozygosity: (n/(n-1)) * 2p(1-p). ~0.4% at pfal's 236 alleles, + -- ~9% at tbru's 12, 100% at the 2-allele floor - which is why the >= 4 filter + -- exists rather than trusting the estimator. + tx_pi AS ( + SELECT t.source, t.gene_source_id, + sum(CASE WHEN t.sev=5 THEN (v.n_alleles/(v.n_alleles-1))*2*v.maf*(1-v.maf) END) AS pi_sum_nonsyn, + sum(CASE WHEN t.sev=7 THEN (v.n_alleles/(v.n_alleles-1))*2*v.maf*(1-v.maf) END) AS pi_sum_syn, + sum((v.n_alleles/(v.n_alleles-1))*2*v.maf*(1-v.maf)) AS pi_sum_all, + count(*) FILTER (WHERE t.sev=5) AS tx_n_missense, + count(*) FILTER (WHERE t.sev=7) AS tx_n_synonymous, + count(*) AS tx_n_loci_pi + FROM tx_locus t JOIN vf v + ON v.sequence_source_id = t.sequence_source_id AND v.location = t.location + WHERE v.n_alleles >= 4 AND v.maf IS NOT NULL + GROUP BY 1,2 + ) + SELECT ':PROJECT_ID' as project_id + , ':ORG_ABBREV' as org_abbrev + , current_timestamp as modification_date + , r.organism + , r.taxon_id + , r.gene_source_id + , r.gene_na_feature_id + , r.chromosome + , r.gene_length + , r.cds_length + , r.transcript_source_id AS rep_transcript_source_id + , gs.syn_sites + , gs.nonsyn_sites + , round((gs.syn_sites/nullif(gs.syn_sites+gs.nonsyn_sites,0))::numeric,4) AS syn_site_fraction + , a.max_called_strain_count + , a.median_called_strain_count + , a.max_alleles + , a.effective_ploidy + , a.avg_call_rate + , a.min_call_rate + , a.n_low_call_rate + , a.n_het_loci + , round((a.n_het_loci::numeric/nullif(a.total_variants,0)),3) AS prop_het_loci + , a.n_loci_pi + , a.n_loci_freq20 + , a.n_loci_freq100 + , a.total_variants + , a.n_coding_loci + , a.n_snv + , a.n_indel + , a.n_mixed + , round((1000.0*a.total_variants/nullif(r.gene_length,0)),2) AS variants_per_kb + , a.n_frameshift + , a.n_nonsense + , a.n_splice_disruptive + , a.n_inframe_indel + , a.n_missense + , a.n_splice_region + , a.n_synonymous + , a.n_utr + , a.n_noncoding_exon + , a.n_intron + , a.n_other + , a.n_lof + , a.n_indel_frameshift + , a.n_impact_high + , a.n_impact_moderate + , a.n_impact_low + , a.n_impact_modifier + , a.most_severe_impact + , p.pc_total_coding_variants + , p.pc_n_missense + , p.pc_n_synonymous + , p.pc_n_lof + , p.pc_n_unclassified + -- frequency bins: NULL, not 0, when no locus cleared the floor + , CASE WHEN a.n_loci_freq20 > 0 THEN a.n_common_raw END AS n_common + , CASE WHEN a.n_loci_freq20 > 0 THEN a.n_very_common_raw END AS n_very_common + , CASE WHEN a.n_loci_freq20 > 0 THEN a.n_missense_common_raw END AS n_missense_common + , CASE WHEN a.n_loci_freq20 > 0 THEN a.n_synonymous_common_raw END AS n_synonymous_common + , CASE WHEN a.n_loci_freq20 > 0 THEN a.n_lof_common_raw END AS n_lof_common + , CASE WHEN a.n_loci_freq20 > 0 THEN p.pc_n_missense_common_raw END AS pc_n_missense_common + , CASE WHEN a.n_loci_freq20 > 0 THEN p.pc_n_lof_common_raw END AS pc_n_lof_common + , CASE WHEN a.n_loci_freq20 > 0 THEN round(a.max_maf_raw::numeric,4) END AS max_minor_allele_frequency + , CASE WHEN a.n_loci_freq100 > 0 THEN a.n_rare_raw END AS n_rare + , CASE WHEN a.n_loci_freq100 > 0 THEN a.n_singleton_raw END AS n_singleton + -- count ratios: a denominator under 5 synonymous loci is noise. NOT a + -- selection statistic - pfal's median count ratio is 2.22 while its median + -- site-normalized piN/piS is 0.512. Label as a raw count ratio on the page. + , CASE WHEN a.n_synonymous >= 5 + THEN round((a.n_missense::numeric/a.n_synonymous),2) END AS nonsyn_syn_ratio_snpeff + , CASE WHEN p.pc_n_synonymous >= 5 + THEN round((p.pc_n_missense::numeric/p.pc_n_synonymous),2) END AS nonsyn_syn_ratio_product_call + , round((se.pi_sum_nonsyn/nullif(gs.nonsyn_sites,0))::numeric,6) AS pi_nonsyn_snpeff + , round((se.pi_sum_syn /nullif(gs.syn_sites,0))::numeric,6) AS pi_syn_snpeff + , CASE WHEN se.tx_n_synonymous >= 5 THEN + round(((se.pi_sum_nonsyn/nullif(gs.nonsyn_sites,0)) + /nullif(se.pi_sum_syn/nullif(gs.syn_sites,0),0))::numeric,3) END AS pi_n_pi_s_snpeff + , round((pp.pi_sum_nonsyn/nullif(gs.nonsyn_sites,0))::numeric,6) AS pi_nonsyn_product_call + , round((pp.pi_sum_syn /nullif(gs.syn_sites,0))::numeric,6) AS pi_syn_product_call + , CASE WHEN pp.tx_n_synonymous >= 5 THEN + round(((pp.pi_sum_nonsyn/nullif(gs.nonsyn_sites,0)) + /nullif(pp.pi_sum_syn/nullif(gs.syn_sites,0),0))::numeric,3) END AS pi_n_pi_s_product_call + , round((se.pi_sum_all/nullif(gs.syn_sites+gs.nonsyn_sites,0))::numeric,6) AS pi_per_site_cds + , se.tx_n_missense AS tx_n_missense_snpeff + , se.tx_n_synonymous AS tx_n_synonymous_snpeff + , se.tx_n_loci_pi AS tx_n_loci_pi_snpeff + , pp.tx_n_missense AS tx_n_missense_product_call + , pp.tx_n_synonymous AS tx_n_synonymous_product_call + FROM rep r + JOIN snp_agg a ON a.gene_source_id = r.gene_source_id + LEFT JOIN pc_agg p ON p.gene_source_id = r.gene_source_id + LEFT JOIN gene_sites gs ON gs.gene_source_id = r.gene_source_id + LEFT JOIN tx_pi se ON se.gene_source_id = r.gene_source_id AND se.source = 'snpeff' + LEFT JOIN tx_pi pp ON pp.gene_source_id = r.gene_source_id AND pp.source = 'product_call'; + + +:DECLARE_PARTITION; diff --git a/Model/lib/psql/webready/orgSpecific/GeneVariationSummary_p_ix.psql b/Model/lib/psql/webready/orgSpecific/GeneVariationSummary_p_ix.psql new file mode 100644 index 0000000000..0f5d1d245b --- /dev/null +++ b/Model/lib/psql/webready/orgSpecific/GeneVariationSummary_p_ix.psql @@ -0,0 +1,3 @@ + CREATE INDEX GeneVarSumm_ix + ON :SCHEMA.GeneVariationSummary_p (org_abbrev, gene_source_id) + ; diff --git a/Model/lib/xml/tuningManager/apiTuningManager.xml b/Model/lib/xml/tuningManager/apiTuningManager.xml index 82c29f7271..be1b0a71db 100644 --- a/Model/lib/xml/tuningManager/apiTuningManager.xml +++ b/Model/lib/xml/tuningManager/apiTuningManager.xml @@ -615,6 +615,365 @@ + + + One row per gene per project, feeding the "Genetic Variation" section of the gene + record. Replaces six attributes retired from geneRecord.xml (total_hts_snps, + hts_nonsynonymous_snps, hts_synonymous_snps, hts_noncoding_snps, + hts_stop_codon_snps, hts_nonsyn_syn_ratio), which are already commented out on + master along with the TranscriptAttributes_p psql that fed them. + + Mirrors Model/lib/psql/webready/orgSpecific/GeneVariationSummary_p.psql. Any edit + here MUST be mirrored there, and vice versa. Full design, validation evidence, and + the biologist-facing help text: + docs/superpowers/specs/2026-08-07-gene-variation-summary-design.md + + THREE THINGS THAT LOOK LIKE STYLE BUT ARE LOAD-BEARING: + + 1. TWO GRAINS. Display counts are per GENE (unioned across transcripts, + most-severe-wins). The pi statistics are confined to the REPRESENTATIVE + longest-CDS transcript, whose id is stored in rep_transcript_source_id. piN/piS + is defined for one CDS; counting variants across all transcripts while + normalizing by one transcript's site counts would let a variant in a + transcript-specific exon enter the numerator while that exon's sites never + enter the denominator. Only 1.2% of pfal genes have >1 transcript today, but + splicing annotation density is an annotation property, not a constant. + + 2. SUPPRESS, NEVER DEGRADE. Every frequency-derived statistic accumulates only over + loci clearing an allele floor (pi >= 4, common/very-common >= 20, rare/singleton + >= 100) and is NULL below it, never 0. Sample size is a PER-LOCUS property: + min(called_strain_count) is 1 in all three loaded organisms and 35% of pfal loci + have fewer than 100 alleles. An organism with one sample plus a reference has 2 alleles, + where MAF can only be 0.5, so pi would compute to a rescaled variant density + carrying no frequency information. Rendering 0.00 there would reproduce the + exact defect of the retired hts_nonsyn_syn_ratio, which reported 0 for genes + with no synonymous sites and so displayed maximal signal as minimal. The + n_loci_* columns publish how many loci actually contributed. + + 3. PLOIDY IS DERIVED, NEVER HARDCODED (total_ploidy_count / called_strain_count). + Measured: pfal 1.01, tbru 2.27, afum 2.01 - afum being a HAPLOID fungus called + as diploid, which inflates its allele denominators 2x. A hardcoded lookup would + encode that calling bug as truth; a measured value self-corrects when the + upstream calling is fixed. + + MAF is a true ALLELE frequency (verified against pfal to within 2e-5 using + total_ploidy_count as the denominator), so 2p(1-p) is valid at any ploidy. But + *_minor_allele_strain_count is a STRAIN count - different units - so "singleton" is + defined on allele copies, round(maf * total_ploidy_count) = 1. + + Source is apidb.VariationEffect ALONE: at gene x locus grain + VariationTranscriptProduct contributes 0 pairs that VariationEffect lacks + (2,905,385 vs 1,691,462, 0 product-only), so the defensive UNION in + VariationAttributes is dead code at this grain. + + Site counts are Nei-Gojobori (1986), derived inline from the genetic code rather + than hardcoded, so the code table is the single source of truth. The pooled + synonymous-site fraction in pfal is 17.49%, NOT the textbook ~25% - an AT-bias + effect worth 1.43x on every gene. Without this normalization the median piN/piS is + 2.0, implying genome-wide positive selection; with it the median is 0.512, the + expected purifying-selection signature. + + COST NOTE: the codon work is streamed per-CDS via LATERAL rather than expanded flat + (see the gene_sites CTE). Measured over all 63,082 genes in unidb_shu_a it is 31 + seconds, so it is NOT the expensive step of this build - the VariationEffect + aggregation is. Do not "optimize" it into a flat expansion or a temp table; that was + measured 40% slower with a much larger peak footprint. + + + + + + + CTA), CGA 1.333. + nbr AS ( + SELECT c.codon, c.aa, p.pos, b.base, + overlay(c.codon placing b.base from p.pos for 1) AS mutated + FROM code c + CROSS JOIN generate_series(1,3) AS p(pos) + CROSS JOIN (VALUES ('A'),('C'),('G'),('T')) AS b(base) + WHERE c.aa <> '*' AND substr(c.codon, p.pos, 1) <> b.base + ), + posf AS ( + SELECT n.codon, n.pos, count(*) FILTER (WHERE m.aa = n.aa)::numeric / 3 AS f + FROM nbr n JOIN code m ON m.codon = n.mutated + GROUP BY 1,2 + ), + codon_sites AS ( + SELECT codon, sum(f) AS syn_sites, 3 - sum(f) AS nonsyn_sites + FROM posf GROUP BY codon + ), + -- representative transcript: longest CDS, deterministic tiebreak + rep AS ( + SELECT DISTINCT ON (gene_source_id, project_id) + gene_source_id, project_id, org_abbrev, organism, taxon_id, chromosome, + gene_na_feature_id, transcript_source_id, + na_feature_id AS rep_na_feature_id, + (gene_end_max - gene_start_min + 1) AS gene_length, cds_length + FROM TranscriptAttributes&1 + ORDER BY gene_source_id, project_id, cds_length DESC NULLS LAST, + transcript_source_id + ), + -- per-gene site counts from the representative CDS. The length filter guards + -- against out-of-frame CDS rows, which would otherwise yield garbage codons + -- (477 of 63,765 rows genome-wide, none in the three loaded organisms). + -- LATERAL, not a flat expansion: postgres streams each CDS's codons through the + -- aggregate instead of materializing one row per codon across the whole genome. + -- Measured on unidb_shu_a over all 63,082 genes: 31s vs 52s for the flat form, + -- identical results (0 disagreements), and a far smaller peak footprint. A + -- PL/pgSQL loop would achieve the same but forces procedural code into both this + -- file and the webready mirror; this stays declarative, so the two copies remain + -- textually comparable. + gene_sites AS ( + SELECT r.gene_source_id, r.project_id, s.syn_sites, s.nonsyn_sites + FROM rep r + JOIN webready.CodingSequence_p cs + ON cs.source_id = r.transcript_source_id + AND cs.project_id = r.project_id + CROSS JOIN LATERAL ( + SELECT sum(k.syn_sites) AS syn_sites, + sum(k.nonsyn_sites) AS nonsyn_sites + FROM regexp_matches(upper(cs.sequence), '.{3}', 'g') AS m(arr) + JOIN codon_sites k ON k.codon = m.arr[1] + ) s + WHERE length(cs.sequence) % 3 = 0 AND length(cs.sequence) >= 6 + ), + sev AS ( + SELECT e.source, t.gene_source_id, t.project_id, e.na_feature_id, + e.sequence_source_id, e.location, + CASE e.effect + WHEN 'frameshift_variant' THEN 1 + WHEN 'stop_gained' THEN 2 + WHEN 'stop_lost' THEN 2 + WHEN 'start_lost' THEN 2 + WHEN 'splice_acceptor_variant' THEN 3 + WHEN 'splice_donor_variant' THEN 3 + WHEN 'conservative_inframe_deletion' THEN 4 + WHEN 'disruptive_inframe_deletion' THEN 4 + WHEN 'conservative_inframe_insertion' THEN 4 + WHEN 'disruptive_inframe_insertion' THEN 4 + WHEN 'inframe_deletion_unnormalized' THEN 4 + WHEN 'inframe_insertion_unnormalized' THEN 4 + WHEN 'missense_variant' THEN 5 + WHEN 'splice_region_variant' THEN 6 + WHEN 'synonymous_variant' THEN 7 + WHEN 'stop_retained_variant' THEN 7 + WHEN 'start_retained_variant' THEN 7 + WHEN '5_prime_UTR_variant' THEN 8 + WHEN '3_prime_UTR_variant' THEN 8 + WHEN '5_prime_UTR_premature_start_codon_gain_variant' THEN 8 + WHEN 'non_coding_transcript_exon_variant' THEN 9 + WHEN 'non_coding_transcript_variant' THEN 9 + WHEN 'intron_variant' THEN 10 + ELSE 11 + END AS sev, + CASE e.impact WHEN 'HIGH' THEN 4 WHEN 'MODERATE' THEN 3 + WHEN 'LOW' THEN 2 WHEN 'MODIFIER' THEN 1 END AS imp + FROM apidb.VariationEffect e + JOIN TranscriptAttributes&1 t ON t.na_feature_id = e.na_feature_id + ), + vf AS ( + SELECT sequence_source_id, location, variant_type, is_coding, call_rate, + called_strain_count, het_strain_count, indel_frame_effect, + total_ploidy_count AS n_alleles, + nullif(greatest(coalesce(snp_minor_allele_frequency,0), + coalesce(indel_minor_allele_frequency,0)),0) AS maf + FROM apidb.VariationFeature + ), + gene_locus AS ( + SELECT source, gene_source_id, project_id, sequence_source_id, location, + min(sev) AS sev, max(imp) AS imp + FROM sev GROUP BY 1,2,3,4,5 + ), + gl AS ( + SELECT g.*, v.variant_type, v.is_coding, v.call_rate, v.called_strain_count, + v.het_strain_count, v.indel_frame_effect, v.n_alleles, v.maf, + round(v.maf * v.n_alleles) AS minor_copies + FROM gene_locus g JOIN vf v + ON v.sequence_source_id = g.sequence_source_id AND v.location = g.location + ), + snp_agg AS ( + SELECT gene_source_id, project_id, + count(*) AS total_variants, + count(*) FILTER (WHERE is_coding=1) AS n_coding_loci, + count(*) FILTER (WHERE variant_type='SNV') AS n_snv, + count(*) FILTER (WHERE variant_type='INDEL') AS n_indel, + count(*) FILTER (WHERE variant_type='MIXED') AS n_mixed, + count(*) FILTER (WHERE sev=1) AS n_frameshift, + count(*) FILTER (WHERE sev=2) AS n_nonsense, + count(*) FILTER (WHERE sev=3) AS n_splice_disruptive, + count(*) FILTER (WHERE sev=4) AS n_inframe_indel, + count(*) FILTER (WHERE sev=5) AS n_missense, + count(*) FILTER (WHERE sev=6) AS n_splice_region, + count(*) FILTER (WHERE sev=7) AS n_synonymous, + count(*) FILTER (WHERE sev=8) AS n_utr, + count(*) FILTER (WHERE sev=9) AS n_noncoding_exon, + count(*) FILTER (WHERE sev=10) AS n_intron, + count(*) FILTER (WHERE sev=11) AS n_other, + count(*) FILTER (WHERE sev<=3) AS n_lof, + count(*) FILTER (WHERE indel_frame_effect='frameshift') AS n_indel_frameshift, + count(*) FILTER (WHERE imp=4) AS n_impact_high, + count(*) FILTER (WHERE imp=3) AS n_impact_moderate, + count(*) FILTER (WHERE imp=2) AS n_impact_low, + count(*) FILTER (WHERE imp=1) AS n_impact_modifier, + CASE max(imp) WHEN 4 THEN 'HIGH' WHEN 3 THEN 'MODERATE' + WHEN 2 THEN 'LOW' WHEN 1 THEN 'MODIFIER' END AS most_severe_impact, + max(called_strain_count) AS max_called_strain_count, + round(percentile_cont(0.5) WITHIN GROUP (ORDER BY called_strain_count)::numeric,0) + AS median_called_strain_count, + max(n_alleles) AS max_alleles, + round(avg(n_alleles/nullif(called_strain_count,0))::numeric,2) AS effective_ploidy, + round(avg(call_rate)::numeric,3) AS avg_call_rate, + round(min(call_rate)::numeric,3) AS min_call_rate, + count(*) FILTER (WHERE call_rate < 0.5) AS n_low_call_rate, + count(*) FILTER (WHERE het_strain_count > 0) AS n_het_loci, + count(*) FILTER (WHERE n_alleles >= 4) AS n_loci_pi, + count(*) FILTER (WHERE n_alleles >= 20) AS n_loci_freq20, + count(*) FILTER (WHERE n_alleles >= 100) AS n_loci_freq100, + count(*) FILTER (WHERE n_alleles >= 20 AND maf > 0.05) AS n_common_raw, + count(*) FILTER (WHERE n_alleles >= 20 AND maf > 0.20) AS n_very_common_raw, + count(*) FILTER (WHERE n_alleles >= 20 AND maf > 0.05 AND sev=5) AS n_missense_common_raw, + count(*) FILTER (WHERE n_alleles >= 20 AND maf > 0.05 AND sev=7) AS n_synonymous_common_raw, + count(*) FILTER (WHERE n_alleles >= 20 AND maf > 0.05 AND sev<=3) AS n_lof_common_raw, + max(maf) FILTER (WHERE n_alleles >= 20) AS max_maf_raw, + count(*) FILTER (WHERE n_alleles >= 100 AND maf <= 0.01) AS n_rare_raw, + count(*) FILTER (WHERE n_alleles >= 100 AND minor_copies = 1) AS n_singleton_raw + FROM gl WHERE source='snpeff' GROUP BY 1,2 + ), + pc_agg AS ( + SELECT gene_source_id, project_id, + count(*) AS pc_total_coding_variants, + count(*) FILTER (WHERE sev=5) AS pc_n_missense, + count(*) FILTER (WHERE sev=7) AS pc_n_synonymous, + count(*) FILTER (WHERE sev<=3) AS pc_n_lof, + count(*) FILTER (WHERE sev=11) AS pc_n_unclassified, + count(*) FILTER (WHERE n_alleles >= 20 AND maf > 0.05 AND sev=5) AS pc_n_missense_common_raw, + count(*) FILTER (WHERE n_alleles >= 20 AND maf > 0.05 AND sev<=3) AS pc_n_lof_common_raw + FROM gl WHERE source='product_call' GROUP BY 1,2 + ), + tx_locus AS ( + SELECT s.source, r.gene_source_id, r.project_id, + s.sequence_source_id, s.location, min(s.sev) AS sev + FROM sev s JOIN rep r ON r.rep_na_feature_id = s.na_feature_id + GROUP BY 1,2,3,4,5 + ), + -- unbiased heterozygosity: (n/(n-1)) * 2p(1-p). The correction is ~0.4% at + -- pfal's 236 alleles but ~9% at tbru's 12, and 100% at the 2-allele floor - + -- which is why the >= 4 filter below exists rather than trusting the estimator. + tx_pi AS ( + SELECT t.source, t.gene_source_id, t.project_id, + sum(CASE WHEN t.sev=5 THEN (v.n_alleles/(v.n_alleles-1))*2*v.maf*(1-v.maf) END) AS pi_sum_nonsyn, + sum(CASE WHEN t.sev=7 THEN (v.n_alleles/(v.n_alleles-1))*2*v.maf*(1-v.maf) END) AS pi_sum_syn, + sum((v.n_alleles/(v.n_alleles-1))*2*v.maf*(1-v.maf)) AS pi_sum_all, + count(*) FILTER (WHERE t.sev=5) AS tx_n_missense, + count(*) FILTER (WHERE t.sev=7) AS tx_n_synonymous, + count(*) AS tx_n_loci_pi + FROM tx_locus t JOIN vf v + ON v.sequence_source_id = t.sequence_source_id AND v.location = t.location + WHERE v.n_alleles >= 4 AND v.maf IS NOT NULL + GROUP BY 1,2,3 + ) + SELECT + r.project_id, r.org_abbrev, r.organism, r.taxon_id, + r.gene_source_id, r.gene_na_feature_id, r.chromosome, + r.gene_length, r.cds_length, + r.transcript_source_id AS rep_transcript_source_id, + gs.syn_sites, gs.nonsyn_sites, + round((gs.syn_sites/nullif(gs.syn_sites+gs.nonsyn_sites,0))::numeric,4) AS syn_site_fraction, + a.max_called_strain_count, a.median_called_strain_count, a.max_alleles, + a.effective_ploidy, a.avg_call_rate, a.min_call_rate, a.n_low_call_rate, + a.n_het_loci, + round((a.n_het_loci::numeric/nullif(a.total_variants,0)),3) AS prop_het_loci, + a.n_loci_pi, a.n_loci_freq20, a.n_loci_freq100, + a.total_variants, a.n_coding_loci, a.n_snv, a.n_indel, a.n_mixed, + round((1000.0*a.total_variants/nullif(r.gene_length,0)),2) AS variants_per_kb, + a.n_frameshift, a.n_nonsense, a.n_splice_disruptive, a.n_inframe_indel, + a.n_missense, a.n_splice_region, a.n_synonymous, a.n_utr, + a.n_noncoding_exon, a.n_intron, a.n_other, + a.n_lof, a.n_indel_frameshift, + a.n_impact_high, a.n_impact_moderate, a.n_impact_low, a.n_impact_modifier, + a.most_severe_impact, + p.pc_total_coding_variants, p.pc_n_missense, p.pc_n_synonymous, + p.pc_n_lof, p.pc_n_unclassified, + -- frequency bins: NULL, not 0, when no locus cleared the floor + CASE WHEN a.n_loci_freq20 > 0 THEN a.n_common_raw END AS n_common, + CASE WHEN a.n_loci_freq20 > 0 THEN a.n_very_common_raw END AS n_very_common, + CASE WHEN a.n_loci_freq20 > 0 THEN a.n_missense_common_raw END AS n_missense_common, + CASE WHEN a.n_loci_freq20 > 0 THEN a.n_synonymous_common_raw END AS n_synonymous_common, + CASE WHEN a.n_loci_freq20 > 0 THEN a.n_lof_common_raw END AS n_lof_common, + CASE WHEN a.n_loci_freq20 > 0 THEN p.pc_n_missense_common_raw END AS pc_n_missense_common, + CASE WHEN a.n_loci_freq20 > 0 THEN p.pc_n_lof_common_raw END AS pc_n_lof_common, + CASE WHEN a.n_loci_freq20 > 0 THEN round(a.max_maf_raw::numeric,4) END AS max_minor_allele_frequency, + CASE WHEN a.n_loci_freq100 > 0 THEN a.n_rare_raw END AS n_rare, + CASE WHEN a.n_loci_freq100 > 0 THEN a.n_singleton_raw END AS n_singleton, + -- count ratios: a denominator under 5 synonymous loci is noise. NOT a + -- selection statistic - pfal's median count ratio is 2.22 while its median + -- site-normalized piN/piS is 0.512. Label it as a raw count ratio on the page. + CASE WHEN a.n_synonymous >= 5 + THEN round((a.n_missense::numeric/a.n_synonymous),2) END AS nonsyn_syn_ratio_snpeff, + CASE WHEN p.pc_n_synonymous >= 5 + THEN round((p.pc_n_missense::numeric/p.pc_n_synonymous),2) END AS nonsyn_syn_ratio_product_call, + round((se.pi_sum_nonsyn/nullif(gs.nonsyn_sites,0))::numeric,6) AS pi_nonsyn_snpeff, + round((se.pi_sum_syn /nullif(gs.syn_sites,0))::numeric,6) AS pi_syn_snpeff, + CASE WHEN se.tx_n_synonymous >= 5 THEN + round(((se.pi_sum_nonsyn/nullif(gs.nonsyn_sites,0)) + /nullif(se.pi_sum_syn/nullif(gs.syn_sites,0),0))::numeric,3) END AS pi_n_pi_s_snpeff, + round((pp.pi_sum_nonsyn/nullif(gs.nonsyn_sites,0))::numeric,6) AS pi_nonsyn_product_call, + round((pp.pi_sum_syn /nullif(gs.syn_sites,0))::numeric,6) AS pi_syn_product_call, + CASE WHEN pp.tx_n_synonymous >= 5 THEN + round(((pp.pi_sum_nonsyn/nullif(gs.nonsyn_sites,0)) + /nullif(pp.pi_sum_syn/nullif(gs.syn_sites,0),0))::numeric,3) END AS pi_n_pi_s_product_call, + round((se.pi_sum_all/nullif(gs.syn_sites+gs.nonsyn_sites,0))::numeric,6) AS pi_per_site_cds, + se.tx_n_missense AS tx_n_missense_snpeff, + se.tx_n_synonymous AS tx_n_synonymous_snpeff, + se.tx_n_loci_pi AS tx_n_loci_pi_snpeff, + pp.tx_n_missense AS tx_n_missense_product_call, + pp.tx_n_synonymous AS tx_n_synonymous_product_call + FROM rep r + JOIN snp_agg a ON a.gene_source_id=r.gene_source_id AND a.project_id=r.project_id + LEFT JOIN pc_agg p ON p.gene_source_id=r.gene_source_id AND p.project_id=r.project_id + LEFT JOIN gene_sites gs ON gs.gene_source_id=r.gene_source_id AND gs.project_id=r.project_id + LEFT JOIN tx_pi se ON se.gene_source_id=r.gene_source_id AND se.project_id=r.project_id + AND se.source='snpeff' + LEFT JOIN tx_pi pp ON pp.gene_source_id=r.gene_source_id AND pp.project_id=r.project_id + AND pp.source='product_call' + ]]> + + + + + + + + + diff --git a/docs/superpowers/specs/2026-08-07-gene-variation-summary-design.md b/docs/superpowers/specs/2026-08-07-gene-variation-summary-design.md new file mode 100644 index 0000000000..cc5c81ce97 --- /dev/null +++ b/docs/superpowers/specs/2026-08-07-gene-variation-summary-design.md @@ -0,0 +1,307 @@ +# GeneVariationSummary — gene-page Genetic Variation section + +Status: design, prototype validated +Prototype: `jbrestel.gene_variation_summary_v3` in `unidb_shu_a` (27,297 genes, 3 organisms) +Supporting: `jbrestel.gene_cds_sites`, `jbrestel.codon_sites` + +## 1. What this replaces + +Six attributes retired from the gene and transcript records — `total_hts_snps`, +`hts_nonsynonymous_snps`, `hts_synonymous_snps`, `hts_noncoding_snps`, +`hts_stop_codon_snps`, `hts_nonsyn_syn_ratio`. They are already commented out on master +(`geneRecord.xml:331-343`, `transcriptRecord.xml`, and the `TranscriptAttributes_p.psql` +that fed them), so this is a rebuild, not a migration. + +Two defects in the old set that must not be carried forward: + +**Synonymous was never measured.** It was a residual: +`total - nonsyn - stop - noncoding`. Anything the old pipeline failed to classify landed +in "synonymous," inflating the denominator and biasing the ratio downward. The retired +values are not a target to reproduce. + +**The ratio reported 0 when undefined.** `case when hts_synonymous_snps = 0 then 0` +displayed *maximal* nonsynonymous signal as the *minimum* value. 3,200 of 5,579 pfal +genes have fewer than 5 synonymous sites, so this was not an edge case. + +## 2. Why the old attributes could not simply be fixed + +They were welded into `TranscriptAttributes_p` / `GeneAttributes_p` — giant +workflow-built flat views. They could not be rebuilt, re-derived, or project-gated +without a full workflow run, so when the SNP data model changed, the only affordable +move was to comment out thirty lines across six files. + +`GeneVariationSummary` is therefore its own tuning table, with `externalDependency` on +the three `apidb.Variation*` tables. It rebuilds when the variation data reloads, and it +is additive — which also lets it land on master ahead of the model change that consumes +it. + +## 3. Grain + +Two grains, each used where it is correct. Mixing them is a bug the prototype had in v2. + +| output | grain | +|---|---| +| display counts | per **gene**, unioned across transcripts, most-severe-wins | +| π and πN/πS | **representative transcript only** (longest CDS) | + +πN/πS is defined for one CDS. Counting variants across all transcripts while normalizing +by one transcript's site counts lets a variant in a transcript-specific exon enter the +numerator while that exon's sites never enter the denominator. + +Only 1.2% of pfal genes have more than one transcript (max 3), and afum has none — but +the grain is declared rather than assumed, because splicing annotation density is an +annotation property and this table is cross-project. +`rep_transcript_source_id` is stored so every π value is attributable to a named CDS. + +Source is `apidb.VariationEffect` alone. At gene×locus grain +`VariationTranscriptProduct` contributes **zero** pairs that `VariationEffect` lacks +(2,905,385 vs 1,691,462, 0 product-only), so the defensive UNION in `VariationAttributes` +is dead code at this grain. + +## 4. Effect classification + +Each (gene, locus) gets exactly **one** class by severity rank, so the eleven class +counts **partition** `total_variants` and add up on the page. Verified at 0 violations, +as does the four-way impact summary. + +A locus that is missense in one transcript and synonymous in another counts as missense — +the same "nonsyn wins" semantics the retired columns used. + +Ranks: frameshift(1) · nonsense(2) · splice-disruptive(3) · inframe-indel(4) · +missense(5) · splice-region(6) · synonymous(7) · UTR(8) · noncoding-exon(9) · intron(10) · +other(11). LOF = ranks 1–3. + +### Two callers, asymmetric + +`snpeff` and `product_call` disagree on 19% of paired gene-associated calls, so both are +reported — but they are not symmetric. `product_call` covers **coding only** (no +intergenic, UTR, or intron) and carries 230,532 `downstream_frameshift` rows with no +syn/nonsyn meaning. + +- caller-independent: variant totals, SNV/indel/MIXED, call rate, frequency bins, het +- `snpeff` only: the eleven-class partition, UTR/intron/noncoding counts, impact summary +- both: missense/synonymous/LOF counts, π, πN/πS + +## 5. Ploidy and sample size — the part that governs display + +Sample size is a **per-locus** property, not per-organism. `min(called_strain_count)` is +1 in all three organisms, and 35% of pfal loci have fewer than 100 alleles. + +Measured, never hardcoded (`total_ploidy_count / called_strain_count`): + +| organism | effective ploidy | strains | max alleles | distinct MAF values | +|---|---|---|---|---| +| pfal3D7 | 1.01 (haploid) | 215 | 236 | 4,110 | +| afumAf293 | 2.01 | 232 | 468 | 4,814 | +| tbruTREU927 | 2.27 (diploid) | **4** | 12 | **18** | + +`afumAf293` is a **haploid fungus being called as diploid** — 17% of its loci carry +heterozygous calls. Almost certainly a variant caller left at its diploid default. It +inflates afum allele denominators 2× and manufactures 1.8M "rare" variants. A hardcoded +ploidy lookup would encode that bug as truth; a measured value self-corrects when the +calling is fixed. **Raise with whoever ran that pipeline** — it is a data-production +issue, not something to paper over here. + +`snp_minor_allele_frequency` is a true **allele** frequency (verified against pfal to +within 2×10⁻⁵ using `total_ploidy_count` as denominator), so `2p(1-p)` is valid at any +ploidy. But `*_minor_allele_strain_count` is a **strain** count — different units — so +"singleton" is defined on allele copies, `round(maf × total_ploidy_count) = 1`. + +π carries the `n/(n-1)` finite-sample correction: ~0.4% at pfal's 236 alleles, ~9% at +tbru's 12. + +### Validity domains — suppress, never degrade + +Each frequency-derived statistic accumulates only over loci clearing an allele floor, and +publishes how many contributed. + +| statistic | floor (alleles) | contributing count | +|---|---|---| +| raw counts (`n_missense`, `n_lof`, …) | none — always valid | — | +| nonsyn/syn count ratio | ≥5 synonymous loci | `n_synonymous` | +| π, πN/πS | 4 (+ ≥5 synonymous loci for the ratio) | `n_loci_pi` | +| common / very-common bins | 20 | `n_loci_freq20` | +| rare / singleton bins | 100 | `n_loci_freq100` | + +Below a floor the value is **NULL, never 0**. An organism with one sample plus a +reference has 2 alleles, where MAF can only be 0.5: π would compute to a rescaled variant +density carrying no frequency information, and `n/(n-1)` = 2 is a 100% correction — the +estimator announcing it should not be used. Rendering `0.00` there would repeat the exact +defect of the retired `hts_nonsyn_syn_ratio`, this time on a statistic users would +reasonably trust. + +Verified behaviour: for tbru **every** frequency bin is blank across all 11,689 genes, +while **every** count is present. A single-sample organism keeps everything the old +section had and simply omits the population genetics. + +## 6. Display design + +**Sample basis first.** A biologist cannot read any of this without knowing what it is +based on. The section opens with strain count, effective ploidy, and mean call rate — not +buried in a tooltip. + +Then, in order: + +1. **Basis** — strains sampled, ploidy, mean call rate +2. **Counts** — total variants, SNV / indel / MIXED, variants per kb +3. **Predicted consequences** — impact summary (HIGH/MODERATE/LOW/MODIFIER), then effect classes +4. **Loss of function** — LOF count, and the common-LOF subset called out separately +5. **Selection** — πN/πS, π per site, common-missense fraction +6. **Continuity** — nonsyn/syn count ratio, explicitly labelled as a raw count ratio + +### The count ratio is not a selection statistic — label it so + +pfal's median nonsyn/syn **count** ratio is 2.22, while its median **site-normalized** +πN/πS is 0.512. A biologist reading 2.22 concludes positive selection; the correct +reading is purifying selection. The count ratio does not divide by mutational +opportunity, and there are roughly 4.7× more nonsynonymous than synonymous sites. + +So πN/πS is the selection statistic. The count ratio is retained only for continuity with +what users remember, and must be labelled as a raw ratio of counts. + +### Help text (biologist-facing) + +**Variants** — Positions in this gene where at least one sequenced strain differs from +the reference. Counted once per position, not once per strain. + +**Strains sampled / ploidy** — How many strains carry a call at these positions, and +whether they were called as haploid or diploid. All statistics below are only as good as +this number. + +**Mean call rate** — Average fraction of strains with a confident call. A low call rate +means few variants may reflect poor coverage rather than genuine conservation — the most +common misreading of variation data. + +**Impact** — Severity predicted by SnpEff. HIGH disrupts the protein (frameshift, +premature stop, lost start, disrupted splice site); MODERATE changes an amino acid; +LOW is synonymous or nearly so; MODIFIER is non-coding. + +**Loss-of-function variants** — Frameshifts, premature stops, and disrupted splice sites. +A **common** LOF variant segregating in wild isolates suggests the gene is dispensable; a +singleton is more likely a sequencing artefact. The two are reported separately for that +reason. + +**π (nucleotide diversity)** — Average probability that two randomly chosen strains +differ at a given site in this CDS. Higher means more diverse. + +**πN/πS** — Nonsynonymous diversity divided by synonymous diversity, each normalized by +the number of sites of that class in *this* gene's codons. Below 1 suggests purifying +selection (amino-acid changes removed); above 1 suggests diversifying or balancing +selection, typical of surface antigens under immune pressure. Most genes fall below 1. + +**Common variants** — Minor allele frequency above 5%. Requires at least 20 sampled +alleles; blank when the sample is too small to tell common from rare. + +**Blank vs zero** — A blank value means *not enough data to calculate*, not zero. Each +statistic requires a minimum number of sampled alleles; below that it is suppressed +rather than shown as an unreliable number. + +That last line is the single most important piece of help text in the section. + +## 7. Validation + +Known *P. falciparum* genes behave as a malaria biologist would expect. Site-normalized +πN/πS, snpeff: + +| gene | | πN/πS | π/site | common missense | call rate | +|---|---|---|---|---|---| +| PF3D7_1133400 | AMA1 | 10.32 | high | 57 / 62 | 0.723 | +| PF3D7_0206800 | MSP2 | 13.49 | high | 32 / 39 | 0.521 | +| PF3D7_0930300 | MSP1 | 1.94 | high | 78 / 94 | 0.687 | +| PF3D7_0304600 | CSP | 3.70 | mid | 14 / 15 | 0.821 | +| PF3D7_0709000 | PfCRT | 24.99 | mid | 12 / 22 | **0.424** | +| PF3D7_1343700 | Kelch13 | 3.67 | low | **1 / 8** | 0.480 | + +The vaccine candidates under balancing selection (AMA1, MSP1, MSP2) top the diversity +ranking with common-missense-dominated spectra. Kelch13, conserved with only rare +artemisinin-resistance mutations, is low and rare-dominated. PfCRT and DHFR show drug- +selection signatures — but note PfCRT's 0.424 call rate, exactly the case where the +confidence indicator earns its place. + +Genome-wide medians land where population genetics expects: πN/πS of 0.512 (pfal), 0.214 +(tbru), 0.185 (afum), all below 1; pfal π of 1.5×10⁻³ per site matches published +*P. falciparum* estimates. + +πN/πS tracks inversely with the number of synonymous sites carrying variation — MSP1 at +1.94 on 28 sites is solid; DHFR's 26.9 rests on one. Hence the ≥5 guard. + +## 8. Site counting (Nei–Gojobori) + +Per codon, per position, the fraction of the 3 possible single-nucleotide changes that +are synonymous. Summed over 3 positions gives (S, N) per codon with S + N = 3. Stop +codons are excluded; a change creating a stop is nonsynonymous. + +Because this depends only on the codon it is a **61-row lookup** derived inline from the +genetic code (not hardcoded, so the code table is the single source of truth). Validated +against known degeneracy: ATG and TGG give 0 synonymous sites, four-fold codons exactly +1.0, two-fold 1/3, and the position-1 cases correctly (TTA 0.667 via TTA↔CTA, CGA 1.333). + +CDS comes from **`webready.CodingSequence_p`**, not from +`substr(spliced_sequence, five_prime_utr_length + 1, cds_length)`. Both were verified in +frame and agree exactly (5,318 of 5,720 pfal genes match; the 402 unmatched are +non-coding genes with no CDS, correctly receiving NULL site counts), but +`CodingSequence_p` removes the `dots.SplicedNaSequence` / `dots.Transcript` joins and a +whole class of frame bug. A length-divisible-by-3 filter guards the 477 of 63,765 +out-of-frame rows genome-wide (none in the loaded organisms). + +### Shape: LATERAL, not a flat expansion + +The codons are streamed per-CDS through a `CROSS JOIN LATERAL` aggregate rather than +expanded into one row per codon genome-wide. Measured over all 63,082 genes in +`unidb_shu_a`: + +| shape | time | rows | disagreements | +|---|---|---|---| +| flat CTE expansion | 51.6 s | 63,082 | — | +| `LATERAL` per-gene aggregate | **30.8 s** | 63,082 | **0** | + +So the codon work is **31 seconds**, not the bottleneck — the `VariationEffect` +aggregation is. Do not "optimize" it back into a flat expansion or a temp table. + +A PL/pgSQL loop would achieve the same streaming, but it forces procedural code into two +files that must mirror each other. Staying declarative keeps the tuningManager and +webready copies textually comparable, which is what makes the mirroring obligation +enforceable by reading. + +## 8a. Partitioning + +`webready.GeneVariationSummary_p` is declaratively partitioned `LIST (org_abbrev)`, via +the standard `:CREATE_AND_POPULATE` / `:DECLARE_PARTITION` directives used by all 62 +`orgSpecific` psql files. Confirmed against the live database: every comparable webready +table (`transcriptattributes_p`, `codingsequence_p`, `genomicseqattributes_p`, and both +CNV tables `genecopynumbers_p` / `chrcopynumbers_p`) is `relkind = 'p'` with +`LIST (org_abbrev)` and 8 partitions. The CNV tables therefore needed no change. + +Leading column order follows the convention `project_id, org_abbrev, modification_date, …` +— the partition key must be present and NOT NULL. + +Minor pre-existing inconsistency worth a cleanup: the branch's corrected +`GeneCopyNumbers_p.psql` emits `modification_date` fifth rather than third, deviating from +that convention. Harmless, but it makes the two CNV files and this one look gratuitously +different. + +### The constant matters more than the per-gene refinement + +Pooled synonymous-site fraction in pfal is **17.49%**, not the textbook ~25% — an AT-bias +effect worth 1.43× on every gene. Per-gene spread is second-order (p25–p75 = 16.9–19.0%, +full range 13.2–24.6%). + +So most of the correction comes from using the right *constant*, which itself can only be +discovered by running this analysis. Per-gene values are retained because the table is +cheap once built (24,822 rows, rebuilt only when the annotation changes) and it removes an +approximation that would otherwise need defending. + +## 9. Known limitations + +- **Jukes–Cantor multiple-hit correction not applied.** Unnecessary at within-species + diversity (π ≈ 10⁻³); would matter for between-species divergence. +- **Nei–Gojobori assumes equal mutation probabilities.** Transitions outnumber + transversions and are more often synonymous at position 3, so S is slightly + underestimated. The modified method with a transition/transversion ratio would refine + this; not worth it for per-gene ranking. +- **tbru rests on 4 strains** (18 distinct MAF values across 571,851 loci). π is computed + where the floor permits, but is noisy; all frequency bins are correctly suppressed. +- **`downstream_of_frameshift_strain_ids`** exists in the live + `apidb.VariationTranscriptProduct` but not the checked-in DDL, 100% null across + 4,595,009 rows. Reconcile DDL or table; overlaps the VCF work if it is meant to carry + per-strain IDs. From 9a3864597a497a8ac26e5e2883bfce2c1a133a2d Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Fri, 7 Aug 2026 11:52:07 -0400 Subject: [PATCH 80/98] Add ungated piN/piS columns for display alongside their denominator The >= 5 synonymous-site guard on pi_n_pi_s is statistically right but blanks 57.5% of pfal genes (3,209 of 5,579), including precisely the most-searched ones. The cause is intrinsic: strongly selected genes accumulate few SYNONYMOUS variants, so AMA1 (4 sites), PfCRT (2) and Kelch13 (3) all suppress while MSP1 (28) survives. A blank reads as "no data" to a biologist. So each caller now has a pair: the guarded column for sorting, searching and filtering, where a 2-site gene would otherwise dominate a descending sort; and an ungated twin for display next to tx_n_synonymous_*, letting the reader discount a thin denominator themselves ("10.32, from 4 synonymous sites"). The ungated column is NOT redundant with the model dividing pi_nonsyn by pi_syn. Those are stored rounded to 6 decimals against a pi of ~1e-3, so a reconstructed ratio drifts up to 0.048 from the correctly computed one and 9% of genes differ (1,484 of 16,453). Two code paths must not produce two different numbers for the same statistic. Verified against unidb_shu_a: 27,297 tuning rows and 5,579 webready pfal rows, with zero differences from the deployed apidbtuning.genevariationsummary on every pre-existing column, and zero differences between the two copies on the new ones. The ungated column is populated for 4,421 pfal genes vs 2,370 guarded, recovering 2,051; where both are present they agree exactly. The 1,158 still blank have no synonymous diversity to divide by, so they are correctly undefined rather than suppressed. Requires a tuning manager rerun to materialize the new columns. Co-Authored-By: Claude Opus 5 --- .../orgSpecific/GeneVariationSummary_p.psql | 15 +++++++ .../xml/tuningManager/apiTuningManager.xml | 18 ++++++++ ...026-08-07-gene-variation-summary-design.md | 44 ++++++++++++++++--- 3 files changed, 70 insertions(+), 7 deletions(-) diff --git a/Model/lib/psql/webready/orgSpecific/GeneVariationSummary_p.psql b/Model/lib/psql/webready/orgSpecific/GeneVariationSummary_p.psql index 7035d4e214..bb6e058833 100644 --- a/Model/lib/psql/webready/orgSpecific/GeneVariationSummary_p.psql +++ b/Model/lib/psql/webready/orgSpecific/GeneVariationSummary_p.psql @@ -344,14 +344,29 @@ THEN round((p.pc_n_missense::numeric/p.pc_n_synonymous),2) END AS nonsyn_syn_ratio_product_call , round((se.pi_sum_nonsyn/nullif(gs.nonsyn_sites,0))::numeric,6) AS pi_nonsyn_snpeff , round((se.pi_sum_syn /nullif(gs.syn_sites,0))::numeric,6) AS pi_syn_snpeff + -- GUARDED twin: for sorting/searching/filtering, where a gene with 2 + -- synonymous sites would otherwise dominate a descending sort. , CASE WHEN se.tx_n_synonymous >= 5 THEN round(((se.pi_sum_nonsyn/nullif(gs.nonsyn_sites,0)) /nullif(se.pi_sum_syn/nullif(gs.syn_sites,0),0))::numeric,3) END AS pi_n_pi_s_snpeff + -- UNGATED twin: for DISPLAY next to tx_n_synonymous_snpeff, so the reader can + -- discount a thin denominator ("10.32, from 4 synonymous sites"). The guard + -- blanks 57.5% of pfal genes including the most-searched ones, because + -- strongly selected genes accumulate few SYNONYMOUS variants: AMA1 (4 sites), + -- PfCRT (2), Kelch13 (3) all suppress while MSP1 (28) survives, and a blank + -- reads as "no data". Cannot be replaced by the model dividing pi_nonsyn by + -- pi_syn - those are stored rounded to 6 decimals against a pi of ~1e-3, so a + -- reconstructed ratio drifts up to 0.048 (9% of genes differ), and two code + -- paths must not yield two different numbers for the same statistic. + , round(((se.pi_sum_nonsyn/nullif(gs.nonsyn_sites,0)) + /nullif(se.pi_sum_syn/nullif(gs.syn_sites,0),0))::numeric,3) AS pi_n_pi_s_snpeff_ungated , round((pp.pi_sum_nonsyn/nullif(gs.nonsyn_sites,0))::numeric,6) AS pi_nonsyn_product_call , round((pp.pi_sum_syn /nullif(gs.syn_sites,0))::numeric,6) AS pi_syn_product_call , CASE WHEN pp.tx_n_synonymous >= 5 THEN round(((pp.pi_sum_nonsyn/nullif(gs.nonsyn_sites,0)) /nullif(pp.pi_sum_syn/nullif(gs.syn_sites,0),0))::numeric,3) END AS pi_n_pi_s_product_call + , round(((pp.pi_sum_nonsyn/nullif(gs.nonsyn_sites,0)) + /nullif(pp.pi_sum_syn/nullif(gs.syn_sites,0),0))::numeric,3) AS pi_n_pi_s_product_call_ungated , round((se.pi_sum_all/nullif(gs.syn_sites+gs.nonsyn_sites,0))::numeric,6) AS pi_per_site_cds , se.tx_n_missense AS tx_n_missense_snpeff , se.tx_n_synonymous AS tx_n_synonymous_snpeff diff --git a/Model/lib/xml/tuningManager/apiTuningManager.xml b/Model/lib/xml/tuningManager/apiTuningManager.xml index be1b0a71db..4d6683fcb2 100644 --- a/Model/lib/xml/tuningManager/apiTuningManager.xml +++ b/Model/lib/xml/tuningManager/apiTuningManager.xml @@ -938,14 +938,32 @@ THEN round((p.pc_n_missense::numeric/p.pc_n_synonymous),2) END AS nonsyn_syn_ratio_product_call, round((se.pi_sum_nonsyn/nullif(gs.nonsyn_sites,0))::numeric,6) AS pi_nonsyn_snpeff, round((se.pi_sum_syn /nullif(gs.syn_sites,0))::numeric,6) AS pi_syn_snpeff, + -- GUARDED twin: for sorting, searching and filtering, where a gene with 2 + -- synonymous sites would otherwise dominate a descending sort. CASE WHEN se.tx_n_synonymous >= 5 THEN round(((se.pi_sum_nonsyn/nullif(gs.nonsyn_sites,0)) /nullif(se.pi_sum_syn/nullif(gs.syn_sites,0),0))::numeric,3) END AS pi_n_pi_s_snpeff, + -- UNGATED twin: for DISPLAY, rendered next to tx_n_synonymous_snpeff so the + -- reader can discount a thin denominator themselves ("10.32, from 4 synonymous + -- sites"). Needed because the guard blanks 57.5% of pfal genes, including + -- precisely the most-searched ones - strongly selected genes accumulate few + -- SYNONYMOUS variants, so AMA1 (4 sites), PfCRT (2) and Kelch13 (3) all + -- suppress while MSP1 (28) survives. A blank reads as "no data". + -- + -- This column cannot be dropped in favour of the model dividing + -- pi_nonsyn_snpeff by pi_syn_snpeff: those are stored rounded to 6 decimals + -- and pi is ~1e-3, so a reconstructed ratio drifts up to 0.048 from this one + -- (9% of genes differ). Two code paths must not yield two different numbers + -- for the same statistic. + round(((se.pi_sum_nonsyn/nullif(gs.nonsyn_sites,0)) + /nullif(se.pi_sum_syn/nullif(gs.syn_sites,0),0))::numeric,3) AS pi_n_pi_s_snpeff_ungated, round((pp.pi_sum_nonsyn/nullif(gs.nonsyn_sites,0))::numeric,6) AS pi_nonsyn_product_call, round((pp.pi_sum_syn /nullif(gs.syn_sites,0))::numeric,6) AS pi_syn_product_call, CASE WHEN pp.tx_n_synonymous >= 5 THEN round(((pp.pi_sum_nonsyn/nullif(gs.nonsyn_sites,0)) /nullif(pp.pi_sum_syn/nullif(gs.syn_sites,0),0))::numeric,3) END AS pi_n_pi_s_product_call, + round(((pp.pi_sum_nonsyn/nullif(gs.nonsyn_sites,0)) + /nullif(pp.pi_sum_syn/nullif(gs.syn_sites,0),0))::numeric,3) AS pi_n_pi_s_product_call_ungated, round((se.pi_sum_all/nullif(gs.syn_sites+gs.nonsyn_sites,0))::numeric,6) AS pi_per_site_cds, se.tx_n_missense AS tx_n_missense_snpeff, se.tx_n_synonymous AS tx_n_synonymous_snpeff, diff --git a/docs/superpowers/specs/2026-08-07-gene-variation-summary-design.md b/docs/superpowers/specs/2026-08-07-gene-variation-summary-design.md index cc5c81ce97..1d898486f9 100644 --- a/docs/superpowers/specs/2026-08-07-gene-variation-summary-design.md +++ b/docs/superpowers/specs/2026-08-07-gene-variation-summary-design.md @@ -203,14 +203,44 @@ That last line is the single most important piece of help text in the section. Known *P. falciparum* genes behave as a malaria biologist would expect. Site-normalized πN/πS, snpeff: -| gene | | πN/πS | π/site | common missense | call rate | +Values below are as **delivered** (the ≥5 synonymous-site guard applied). The +parenthesised figures are the ungated values, shown only to make the guard's effect +visible — they are not what the page displays. + +| gene | | πN/πS delivered | syn sites | common missense | call rate | |---|---|---|---|---|---| -| PF3D7_1133400 | AMA1 | 10.32 | high | 57 / 62 | 0.723 | -| PF3D7_0206800 | MSP2 | 13.49 | high | 32 / 39 | 0.521 | -| PF3D7_0930300 | MSP1 | 1.94 | high | 78 / 94 | 0.687 | -| PF3D7_0304600 | CSP | 3.70 | mid | 14 / 15 | 0.821 | -| PF3D7_0709000 | PfCRT | 24.99 | mid | 12 / 22 | **0.424** | -| PF3D7_1343700 | Kelch13 | 3.67 | low | **1 / 8** | 0.480 | +| PF3D7_0304600 | CSP | 3.70 | 12 | 14 / 15 | 0.821 | +| PF3D7_0930300 | MSP1 | 1.94 | 28 | 77 / 94 | 0.687 | +| PF3D7_0206800 | MSP2 | 13.49 | 7 | 32 / 39 | 0.521 | +| PF3D7_1133400 | AMA1 | **blank** (10.32) | 4 | 57 / 62 | 0.723 | +| PF3D7_0709000 | PfCRT | **blank** (24.99) | 2 | 12 / 22 | **0.424** | +| PF3D7_1343700 | Kelch13 | **blank** (3.67) | 3 | **1 / 8** | 0.480 | + +### Open question: the ≥5 guard blanks the most-looked-at genes + +The guard is statistically right and biologically inconvenient, for the same reason: +strongly selected genes accumulate **few synonymous** variants, so the genes a malaria +biologist searches for first are exactly the ones whose denominator is too thin to trust. +AMA1 (4 synonymous sites), PfCRT (2), and Kelch13 (3) are all suppressed; MSP1 (28) +survives. + +Scale of the effect on pfal: **πN/πS is blank for 57.5%** of genes (3,209 of 5,579). +Median synonymous sites per gene is 3. Relaxing to ≥3 would recover 895 genes; to ≥1, +2,258. + +Three options, not yet decided: + +1. **Keep ≥5.** Statistically honest, but a biologist looking up AMA1 sees nothing and + concludes the site has no data. +2. **Relax to ≥3.** Recovers AMA1 and Kelch13; still blanks PfCRT. Arbitrary. +3. **Show the value with its denominator.** `tx_n_synonymous_snpeff` is already a column, + so the page can render "10.32 (from 4 synonymous sites)" and let the reader discount + it. This is the only option that neither hides the signal nor implies false precision, + and it fits the "explaining is key" requirement better than a blank. + +Recommendation is (3) for display while keeping the guarded column for sorting and +searching, where a thin-denominator outlier would otherwise dominate a sort. That needs a +decision before the record wiring, since it changes which columns the page reads. The vaccine candidates under balancing selection (AMA1, MSP1, MSP2) top the diversity ranking with common-missense-dominated spectra. Kelch13, conserved with only rare From 61e49a338833b4a93732ab153bdc8d31f3d1bc9f Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Fri, 7 Aug 2026 12:07:57 -0400 Subject: [PATCH 81/98] Fix: reference TranscriptAttributes unversioned, not with &1 The tuning manager substitutes &1 with the version of the table BEING BUILT, so TranscriptAttributes&1 resolved to transcriptattributes1125 - a version that does not exist, since TranscriptAttributes is at 1118. Internal dependencies are referenced by their unversioned name and resolve through the apidbtuning view; &1 belongs only on the table being created and its own indexes. VariationAttributes already had this right and was the precedent I should have followed. Broke the tuning run with: ERROR: relation "transcriptattributes1125" does not exist LINE 51: FROM TranscriptAttributes1125 My verification missed it because the test harness rewrote TranscriptAttributes&1 to apidbtuning.TranscriptAttributes as part of emulating the substitution - so it silently tested the correct form while the shipped XML held the broken one. The harness now asserts that no dependency carries &1 after the built table's own name is substituted, and fails rather than repairing it. No value changes: 27,297 rows, zero differences from the previous build on every column including syn_sites, both piN/piS variants, and rep_transcript_source_id. Co-Authored-By: Claude Opus 5 --- Model/lib/xml/tuningManager/apiTuningManager.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Model/lib/xml/tuningManager/apiTuningManager.xml b/Model/lib/xml/tuningManager/apiTuningManager.xml index 4d6683fcb2..f5067ed27d 100644 --- a/Model/lib/xml/tuningManager/apiTuningManager.xml +++ b/Model/lib/xml/tuningManager/apiTuningManager.xml @@ -733,7 +733,7 @@ gene_na_feature_id, transcript_source_id, na_feature_id AS rep_na_feature_id, (gene_end_max - gene_start_min + 1) AS gene_length, cds_length - FROM TranscriptAttributes&1 + FROM TranscriptAttributes ORDER BY gene_source_id, project_id, cds_length DESC NULLS LAST, transcript_source_id ), @@ -793,7 +793,7 @@ CASE e.impact WHEN 'HIGH' THEN 4 WHEN 'MODERATE' THEN 3 WHEN 'LOW' THEN 2 WHEN 'MODIFIER' THEN 1 END AS imp FROM apidb.VariationEffect e - JOIN TranscriptAttributes&1 t ON t.na_feature_id = e.na_feature_id + JOIN TranscriptAttributes t ON t.na_feature_id = e.na_feature_id ), vf AS ( SELECT sequence_source_id, location, variant_type, is_coding, call_rate, From ed68d8127c318a43a6f89001b38dbb295e62296b Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Fri, 7 Aug 2026 12:27:40 -0400 Subject: [PATCH 82/98] Wire the gene-record Genetic Variation attributes Surfaces apidbtuning.GeneVariationSummary on the gene record page and in the tabular results view, replacing the six retired hts_* attributes. Genes and transcripts are a special case: the tabular results view is TranscriptRecordClass while the record page is GeneRecordClass. So each side gets its own attribute query over the same tuning table, and the ontology scope column is what actually routes them - GeneRecordClass rows carry 'record', the TranscriptRecordClass rows carry 'results'. This is not duplication for its own sake; it is the mechanism, and it is why the retired attributes existed as both hts_* and gene_hts_*. The split is also used deliberately for piN/piS. The results view sorts on the GUARDED column (>= 5 synonymous sites), because a gene resting on 2 synonymous sites would otherwise top every descending sort. The record page shows the UNGATED value next to pi_syn_sites_used, so a reader can discount a thin denominator instead of seeing a blank - the guard alone hides 57.5% of pfal genes, including AMA1, PfCRT and Kelch13, because strongly selected genes accumulate few synonymous variants. Record page carries the full set (32 columns: basis, counts, impact summary, effect classes, loss-of-function, selection, second caller, and the raw count ratio for continuity). Results view carries the sortable subset (13 columns) so the table stays usable. Sample basis leads the section on purpose - strains sampled, ploidy, mean call rate. Mean call rate across pfal is 0.552, so "few variants" frequently means poor coverage rather than conservation, and that is the most common misreading of variation data. Help text spells out that a blank means "not enough data", never zero; the raw count ratio is explicitly labelled as not a measure of selection, since its pfal median is 2.22 against a site-normalized piN/piS median of 0.512. Both queries are kept separate from the Bfmv flat view deliberately, so they can change without a Bfmv rebuild - being welded into it is why the old attributes were commented out rather than fixed. Verified against unidb_shu_a with %%PARTITION_KEYS%% substituted: both queries execute (5,720 gene rows, 5,791 transcript rows for pfal), every referenced tuning column exists, and declared elements match actual output columns exactly in both directions. The LEFT JOIN populates 5,579 of 5,720 pfal genes, leaving 141 null for genes with no variants. All four XML files parse; the 40 new ontology rows all have 14 fields. Needs 'wb ontology' rather than 'wb model' - individuals.txt changed. Co-Authored-By: Claude Opus 5 --- .../model/records/geneAttributeQueries.xml | 96 +++++++++++++++++ Model/lib/wdk/model/records/geneRecord.xml | 100 ++++++++++++++++++ .../records/transcriptAttributeQueries.xml | 55 ++++++++++ .../wdk/model/records/transcriptRecord.xml | 40 +++++++ Model/lib/wdk/ontology/individuals.txt | 40 +++++++ 5 files changed, 331 insertions(+) diff --git a/Model/lib/wdk/model/records/geneAttributeQueries.xml b/Model/lib/wdk/model/records/geneAttributeQueries.xml index 8e1e2dbb0c..5d998d3dff 100644 --- a/Model/lib/wdk/model/records/geneAttributeQueries.xml +++ b/Model/lib/wdk/model/records/geneAttributeQueries.xml @@ -146,6 +146,102 @@ WHERE ga.org_abbrev IN (%%PARTITION_KEYS%%) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/Model/lib/wdk/model/records/geneRecord.xml b/Model/lib/wdk/model/records/geneRecord.xml index 5304a70b57..9a506df7f2 100644 --- a/Model/lib/wdk/model/records/geneRecord.xml +++ b/Model/lib/wdk/model/records/geneRecord.xml @@ -393,6 +393,106 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/Model/lib/wdk/model/records/transcriptAttributeQueries.xml b/Model/lib/wdk/model/records/transcriptAttributeQueries.xml index 363e68dc80..b200036a55 100644 --- a/Model/lib/wdk/model/records/transcriptAttributeQueries.xml +++ b/Model/lib/wdk/model/records/transcriptAttributeQueries.xml @@ -589,6 +589,61 @@ ELSE 'N/A' end as apollo_link_out + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/Model/lib/wdk/model/records/transcriptRecord.xml b/Model/lib/wdk/model/records/transcriptRecord.xml index 946385a18a..aee363de13 100644 --- a/Model/lib/wdk/model/records/transcriptRecord.xml +++ b/Model/lib/wdk/model/records/transcriptRecord.xml @@ -738,6 +738,46 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Date: Fri, 7 Aug 2026 12:39:26 -0400 Subject: [PATCH 83/98] Group the gene variation attributes into six record subsections All 30 record-page attributes were flat under DNA polymorphism, which is unreadable at that count. They now sit in six collapsible subsections following the display order in the design spec: Sample Basis, Variant Counts, Predicted Impact, Predicted Consequences, Loss of Function, Diversity and Selection. Three groupings are deliberate rather than cosmetic: - Sample Basis is its own group and comes first. Mean call rate across pfal is 0.552, so "few variants" frequently reflects coverage rather than conservation. That has to be read before anything below it. - "Synonymous Sites Behind piN/piS" sits immediately after piN/piS inside Diversity and Selection, so the denominator cannot be visually separated from the value it qualifies. That adjacency is the entire reason we chose display-with-denominator over suppression. - Predicted Impact is kept separate from Predicted Consequences. Impact is the four-bucket triage summary, consequences the SO-term detail; merging them buries the four numbers most readers actually want. Category ids are prefixed GeneVariation* rather than reusing the branch's Variation*Category names: category nodes carry no recordClassName, so identical ids would have merged these groups with the variation record's SNP Alleles / Indel Alleles / Strain Statistics. Display orders 10-15 place them after those three under the shared DNA polymorphism node. The results-view attributes are filed into the same six groups so the attribute picker is organized too, not just the record page. Verified after wb ontology on jbrestel.plasmodb.org: all six categories present in /service/ontologies/Categories with the expected children, all six render as wdk-RecordSubsection blocks on the gene page, and no transcript-scope gene_* attribute leaks onto the record page. On AMA1 all three count partitions independently sum to the 104 total (95+9+0 SNV/indel/mixed; 1+82+4+17 impact; 82+4+1+17 consequences), so the invariant is visible to the reader. Build clean, both error logs silent, no field-count or duplicate-id regression in individuals.txt. Co-Authored-By: Claude Opus 5 --- Model/lib/wdk/ontology/individuals.txt | 86 ++++++++++++++------------ 1 file changed, 46 insertions(+), 40 deletions(-) diff --git a/Model/lib/wdk/ontology/individuals.txt b/Model/lib/wdk/ontology/individuals.txt index 0ff68d7c9b..b7e53b0d6c 100644 --- a/Model/lib/wdk/ontology/individuals.txt +++ b/Model/lib/wdk/ontology/individuals.txt @@ -467,46 +467,52 @@ GeneRecordClasses.GeneRecordClass.hts_synonymous_snps http://edamontology.org/to GeneRecordClasses.GeneRecordClass.hts_noncoding_snps http://edamontology.org/topic_2885 DNA Polymorphism GeneRecordClasses.GeneRecordClass attribute hts_noncoding_snps gene 1 results record download GeneRecordClasses.GeneRecordClass.hts_stop_codon_snps http://edamontology.org/topic_2885 DNA Polymorphism GeneRecordClasses.GeneRecordClass attribute hts_stop_codon_snps gene 1 results record download GeneRecordClasses.GeneRecordClass.hts_nonsyn_syn_ratio http://edamontology.org/topic_2885 DNA Polymorphism GeneRecordClasses.GeneRecordClass attribute hts_nonsyn_syn_ratio gene 1 results record download -GeneRecordClasses.GeneRecordClass.variation_strains_sampled http://edamontology.org/topic_2885 DNA Polymorphism GeneRecordClasses.GeneRecordClass attribute variation_strains_sampled gene 1 record download -GeneRecordClasses.GeneRecordClass.variation_effective_ploidy http://edamontology.org/topic_2885 DNA Polymorphism GeneRecordClasses.GeneRecordClass attribute variation_effective_ploidy gene 2 record download -GeneRecordClasses.GeneRecordClass.variation_call_rate http://edamontology.org/topic_2885 DNA Polymorphism GeneRecordClasses.GeneRecordClass attribute variation_call_rate gene 3 record download -GeneRecordClasses.GeneRecordClass.total_variants http://edamontology.org/topic_2885 DNA Polymorphism GeneRecordClasses.GeneRecordClass attribute total_variants gene 4 record download -GeneRecordClasses.GeneRecordClass.variants_per_kb http://edamontology.org/topic_2885 DNA Polymorphism GeneRecordClasses.GeneRecordClass attribute variants_per_kb gene 5 record download -GeneRecordClasses.GeneRecordClass.variant_snvs http://edamontology.org/topic_2885 DNA Polymorphism GeneRecordClasses.GeneRecordClass attribute variant_snvs gene 6 record download -GeneRecordClasses.GeneRecordClass.variant_indels http://edamontology.org/topic_2885 DNA Polymorphism GeneRecordClasses.GeneRecordClass attribute variant_indels gene 7 record download -GeneRecordClasses.GeneRecordClass.variant_mixed http://edamontology.org/topic_2885 DNA Polymorphism GeneRecordClasses.GeneRecordClass attribute variant_mixed gene 8 record download -GeneRecordClasses.GeneRecordClass.variants_impact_high http://edamontology.org/topic_2885 DNA Polymorphism GeneRecordClasses.GeneRecordClass attribute variants_impact_high gene 9 record download -GeneRecordClasses.GeneRecordClass.variants_impact_moderate http://edamontology.org/topic_2885 DNA Polymorphism GeneRecordClasses.GeneRecordClass attribute variants_impact_moderate gene 10 record download -GeneRecordClasses.GeneRecordClass.variants_impact_low http://edamontology.org/topic_2885 DNA Polymorphism GeneRecordClasses.GeneRecordClass attribute variants_impact_low gene 11 record download -GeneRecordClasses.GeneRecordClass.variants_impact_modifier http://edamontology.org/topic_2885 DNA Polymorphism GeneRecordClasses.GeneRecordClass attribute variants_impact_modifier gene 12 record download -GeneRecordClasses.GeneRecordClass.variants_missense http://edamontology.org/topic_2885 DNA Polymorphism GeneRecordClasses.GeneRecordClass attribute variants_missense gene 13 record download -GeneRecordClasses.GeneRecordClass.variants_synonymous http://edamontology.org/topic_2885 DNA Polymorphism GeneRecordClasses.GeneRecordClass attribute variants_synonymous gene 14 record download -GeneRecordClasses.GeneRecordClass.variants_nonsense http://edamontology.org/topic_2885 DNA Polymorphism GeneRecordClasses.GeneRecordClass attribute variants_nonsense gene 15 record download -GeneRecordClasses.GeneRecordClass.variants_frameshift http://edamontology.org/topic_2885 DNA Polymorphism GeneRecordClasses.GeneRecordClass attribute variants_frameshift gene 16 record download -GeneRecordClasses.GeneRecordClass.variants_splice_disruptive http://edamontology.org/topic_2885 DNA Polymorphism GeneRecordClasses.GeneRecordClass attribute variants_splice_disruptive gene 17 record download -GeneRecordClasses.GeneRecordClass.variants_inframe_indel http://edamontology.org/topic_2885 DNA Polymorphism GeneRecordClasses.GeneRecordClass attribute variants_inframe_indel gene 18 record download -GeneRecordClasses.GeneRecordClass.variants_utr http://edamontology.org/topic_2885 DNA Polymorphism GeneRecordClasses.GeneRecordClass attribute variants_utr gene 19 record download -GeneRecordClasses.GeneRecordClass.variants_intron http://edamontology.org/topic_2885 DNA Polymorphism GeneRecordClasses.GeneRecordClass attribute variants_intron gene 20 record download -GeneRecordClasses.GeneRecordClass.variants_lof http://edamontology.org/topic_2885 DNA Polymorphism GeneRecordClasses.GeneRecordClass attribute variants_lof gene 21 record download -GeneRecordClasses.GeneRecordClass.variants_lof_common http://edamontology.org/topic_2885 DNA Polymorphism GeneRecordClasses.GeneRecordClass attribute variants_lof_common gene 22 record download -GeneRecordClasses.GeneRecordClass.pi_per_site http://edamontology.org/topic_2885 DNA Polymorphism GeneRecordClasses.GeneRecordClass attribute pi_per_site gene 23 record download -GeneRecordClasses.GeneRecordClass.pi_n_pi_s http://edamontology.org/topic_2885 DNA Polymorphism GeneRecordClasses.GeneRecordClass attribute pi_n_pi_s gene 24 record download -GeneRecordClasses.GeneRecordClass.pi_syn_sites_used http://edamontology.org/topic_2885 DNA Polymorphism GeneRecordClasses.GeneRecordClass attribute pi_syn_sites_used gene 25 record download -GeneRecordClasses.GeneRecordClass.variants_common http://edamontology.org/topic_2885 DNA Polymorphism GeneRecordClasses.GeneRecordClass attribute variants_common gene 26 record download -GeneRecordClasses.GeneRecordClass.variants_missense_common http://edamontology.org/topic_2885 DNA Polymorphism GeneRecordClasses.GeneRecordClass attribute variants_missense_common gene 27 record download -GeneRecordClasses.GeneRecordClass.variants_singleton http://edamontology.org/topic_2885 DNA Polymorphism GeneRecordClasses.GeneRecordClass attribute variants_singleton gene 28 record download -GeneRecordClasses.GeneRecordClass.pi_n_pi_s_product_call http://edamontology.org/topic_2885 DNA Polymorphism GeneRecordClasses.GeneRecordClass attribute pi_n_pi_s_product_call gene 29 record download -GeneRecordClasses.GeneRecordClass.nonsyn_syn_count_ratio http://edamontology.org/topic_2885 DNA Polymorphism GeneRecordClasses.GeneRecordClass attribute nonsyn_syn_count_ratio gene 30 record download -TranscriptRecordClasses.TranscriptRecordClass.gene_variation_strains_sampled http://edamontology.org/topic_2885 DNA Polymorphism TranscriptRecordClasses.TranscriptRecordClass attribute gene_variation_strains_sampled gene 1 results download -TranscriptRecordClasses.TranscriptRecordClass.gene_variation_call_rate http://edamontology.org/topic_2885 DNA Polymorphism TranscriptRecordClasses.TranscriptRecordClass attribute gene_variation_call_rate gene 2 results download -TranscriptRecordClasses.TranscriptRecordClass.gene_total_variants http://edamontology.org/topic_2885 DNA Polymorphism TranscriptRecordClasses.TranscriptRecordClass attribute gene_total_variants gene 3 results download -TranscriptRecordClasses.TranscriptRecordClass.gene_variants_per_kb http://edamontology.org/topic_2885 DNA Polymorphism TranscriptRecordClasses.TranscriptRecordClass attribute gene_variants_per_kb gene 4 results download -TranscriptRecordClasses.TranscriptRecordClass.gene_variants_impact_high http://edamontology.org/topic_2885 DNA Polymorphism TranscriptRecordClasses.TranscriptRecordClass attribute gene_variants_impact_high gene 5 results download -TranscriptRecordClasses.TranscriptRecordClass.gene_variants_lof http://edamontology.org/topic_2885 DNA Polymorphism TranscriptRecordClasses.TranscriptRecordClass attribute gene_variants_lof gene 6 results download -TranscriptRecordClasses.TranscriptRecordClass.gene_variants_lof_common http://edamontology.org/topic_2885 DNA Polymorphism TranscriptRecordClasses.TranscriptRecordClass attribute gene_variants_lof_common gene 7 results download -TranscriptRecordClasses.TranscriptRecordClass.gene_variants_missense_common http://edamontology.org/topic_2885 DNA Polymorphism TranscriptRecordClasses.TranscriptRecordClass attribute gene_variants_missense_common gene 8 results download -TranscriptRecordClasses.TranscriptRecordClass.gene_pi_per_site http://edamontology.org/topic_2885 DNA Polymorphism TranscriptRecordClasses.TranscriptRecordClass attribute gene_pi_per_site gene 9 results download -TranscriptRecordClasses.TranscriptRecordClass.gene_pi_n_pi_s http://edamontology.org/topic_2885 DNA Polymorphism TranscriptRecordClasses.TranscriptRecordClass attribute gene_pi_n_pi_s gene 10 results download +GeneVariationBasisCategory http://edamontology.org/topic_2885 DNA polymorphism GeneVariationBasisCategory Sample Basis 10 +GeneVariationCountsCategory http://edamontology.org/topic_2885 DNA polymorphism GeneVariationCountsCategory Variant Counts 11 +GeneVariationImpactCategory http://edamontology.org/topic_2885 DNA polymorphism GeneVariationImpactCategory Predicted Impact 12 +GeneVariationConsequenceCategory http://edamontology.org/topic_2885 DNA polymorphism GeneVariationConsequenceCategory Predicted Consequences 13 +GeneVariationLofCategory http://edamontology.org/topic_2885 DNA polymorphism GeneVariationLofCategory Loss of Function 14 +GeneVariationSelectionCategory http://edamontology.org/topic_2885 DNA polymorphism GeneVariationSelectionCategory Diversity and Selection 15 +GeneRecordClasses.GeneRecordClass.variation_strains_sampled GeneVariationBasisCategory Sample Basis GeneRecordClasses.GeneRecordClass attribute variation_strains_sampled gene 1 record download +GeneRecordClasses.GeneRecordClass.variation_effective_ploidy GeneVariationBasisCategory Sample Basis GeneRecordClasses.GeneRecordClass attribute variation_effective_ploidy gene 2 record download +GeneRecordClasses.GeneRecordClass.variation_call_rate GeneVariationBasisCategory Sample Basis GeneRecordClasses.GeneRecordClass attribute variation_call_rate gene 3 record download +GeneRecordClasses.GeneRecordClass.total_variants GeneVariationCountsCategory Variant Counts GeneRecordClasses.GeneRecordClass attribute total_variants gene 1 record download +GeneRecordClasses.GeneRecordClass.variants_per_kb GeneVariationCountsCategory Variant Counts GeneRecordClasses.GeneRecordClass attribute variants_per_kb gene 2 record download +GeneRecordClasses.GeneRecordClass.variant_snvs GeneVariationCountsCategory Variant Counts GeneRecordClasses.GeneRecordClass attribute variant_snvs gene 3 record download +GeneRecordClasses.GeneRecordClass.variant_indels GeneVariationCountsCategory Variant Counts GeneRecordClasses.GeneRecordClass attribute variant_indels gene 4 record download +GeneRecordClasses.GeneRecordClass.variant_mixed GeneVariationCountsCategory Variant Counts GeneRecordClasses.GeneRecordClass attribute variant_mixed gene 5 record download +GeneRecordClasses.GeneRecordClass.variants_impact_high GeneVariationImpactCategory Predicted Impact GeneRecordClasses.GeneRecordClass attribute variants_impact_high gene 1 record download +GeneRecordClasses.GeneRecordClass.variants_impact_moderate GeneVariationImpactCategory Predicted Impact GeneRecordClasses.GeneRecordClass attribute variants_impact_moderate gene 2 record download +GeneRecordClasses.GeneRecordClass.variants_impact_low GeneVariationImpactCategory Predicted Impact GeneRecordClasses.GeneRecordClass attribute variants_impact_low gene 3 record download +GeneRecordClasses.GeneRecordClass.variants_impact_modifier GeneVariationImpactCategory Predicted Impact GeneRecordClasses.GeneRecordClass attribute variants_impact_modifier gene 4 record download +GeneRecordClasses.GeneRecordClass.variants_missense GeneVariationConsequenceCategory Predicted Consequences GeneRecordClasses.GeneRecordClass attribute variants_missense gene 1 record download +GeneRecordClasses.GeneRecordClass.variants_synonymous GeneVariationConsequenceCategory Predicted Consequences GeneRecordClasses.GeneRecordClass attribute variants_synonymous gene 2 record download +GeneRecordClasses.GeneRecordClass.variants_nonsense GeneVariationConsequenceCategory Predicted Consequences GeneRecordClasses.GeneRecordClass attribute variants_nonsense gene 3 record download +GeneRecordClasses.GeneRecordClass.variants_frameshift GeneVariationConsequenceCategory Predicted Consequences GeneRecordClasses.GeneRecordClass attribute variants_frameshift gene 4 record download +GeneRecordClasses.GeneRecordClass.variants_splice_disruptive GeneVariationConsequenceCategory Predicted Consequences GeneRecordClasses.GeneRecordClass attribute variants_splice_disruptive gene 5 record download +GeneRecordClasses.GeneRecordClass.variants_inframe_indel GeneVariationConsequenceCategory Predicted Consequences GeneRecordClasses.GeneRecordClass attribute variants_inframe_indel gene 6 record download +GeneRecordClasses.GeneRecordClass.variants_utr GeneVariationConsequenceCategory Predicted Consequences GeneRecordClasses.GeneRecordClass attribute variants_utr gene 7 record download +GeneRecordClasses.GeneRecordClass.variants_intron GeneVariationConsequenceCategory Predicted Consequences GeneRecordClasses.GeneRecordClass attribute variants_intron gene 8 record download +GeneRecordClasses.GeneRecordClass.variants_lof GeneVariationLofCategory Loss of Function GeneRecordClasses.GeneRecordClass attribute variants_lof gene 1 record download +GeneRecordClasses.GeneRecordClass.variants_lof_common GeneVariationLofCategory Loss of Function GeneRecordClasses.GeneRecordClass attribute variants_lof_common gene 2 record download +GeneRecordClasses.GeneRecordClass.pi_per_site GeneVariationSelectionCategory Diversity and Selection GeneRecordClasses.GeneRecordClass attribute pi_per_site gene 1 record download +GeneRecordClasses.GeneRecordClass.pi_n_pi_s GeneVariationSelectionCategory Diversity and Selection GeneRecordClasses.GeneRecordClass attribute pi_n_pi_s gene 2 record download +GeneRecordClasses.GeneRecordClass.pi_syn_sites_used GeneVariationSelectionCategory Diversity and Selection GeneRecordClasses.GeneRecordClass attribute pi_syn_sites_used gene 3 record download +GeneRecordClasses.GeneRecordClass.variants_common GeneVariationSelectionCategory Diversity and Selection GeneRecordClasses.GeneRecordClass attribute variants_common gene 4 record download +GeneRecordClasses.GeneRecordClass.variants_missense_common GeneVariationSelectionCategory Diversity and Selection GeneRecordClasses.GeneRecordClass attribute variants_missense_common gene 5 record download +GeneRecordClasses.GeneRecordClass.variants_singleton GeneVariationSelectionCategory Diversity and Selection GeneRecordClasses.GeneRecordClass attribute variants_singleton gene 6 record download +GeneRecordClasses.GeneRecordClass.pi_n_pi_s_product_call GeneVariationSelectionCategory Diversity and Selection GeneRecordClasses.GeneRecordClass attribute pi_n_pi_s_product_call gene 7 record download +GeneRecordClasses.GeneRecordClass.nonsyn_syn_count_ratio GeneVariationSelectionCategory Diversity and Selection GeneRecordClasses.GeneRecordClass attribute nonsyn_syn_count_ratio gene 8 record download +TranscriptRecordClasses.TranscriptRecordClass.gene_variation_strains_sampled GeneVariationBasisCategory Sample Basis TranscriptRecordClasses.TranscriptRecordClass attribute gene_variation_strains_sampled gene 1 results download +TranscriptRecordClasses.TranscriptRecordClass.gene_variation_call_rate GeneVariationBasisCategory Sample Basis TranscriptRecordClasses.TranscriptRecordClass attribute gene_variation_call_rate gene 2 results download +TranscriptRecordClasses.TranscriptRecordClass.gene_total_variants GeneVariationCountsCategory Variant Counts TranscriptRecordClasses.TranscriptRecordClass attribute gene_total_variants gene 1 results download +TranscriptRecordClasses.TranscriptRecordClass.gene_variants_per_kb GeneVariationCountsCategory Variant Counts TranscriptRecordClasses.TranscriptRecordClass attribute gene_variants_per_kb gene 2 results download +TranscriptRecordClasses.TranscriptRecordClass.gene_variants_impact_high GeneVariationImpactCategory Predicted Impact TranscriptRecordClasses.TranscriptRecordClass attribute gene_variants_impact_high gene 1 results download +TranscriptRecordClasses.TranscriptRecordClass.gene_variants_lof GeneVariationLofCategory Loss of Function TranscriptRecordClasses.TranscriptRecordClass attribute gene_variants_lof gene 1 results download +TranscriptRecordClasses.TranscriptRecordClass.gene_variants_lof_common GeneVariationLofCategory Loss of Function TranscriptRecordClasses.TranscriptRecordClass attribute gene_variants_lof_common gene 2 results download +TranscriptRecordClasses.TranscriptRecordClass.gene_pi_per_site GeneVariationSelectionCategory Diversity and Selection TranscriptRecordClasses.TranscriptRecordClass attribute gene_pi_per_site gene 1 results download +TranscriptRecordClasses.TranscriptRecordClass.gene_pi_n_pi_s GeneVariationSelectionCategory Diversity and Selection TranscriptRecordClasses.TranscriptRecordClass attribute gene_pi_n_pi_s gene 2 results download +TranscriptRecordClasses.TranscriptRecordClass.gene_variants_missense_common GeneVariationSelectionCategory Diversity and Selection TranscriptRecordClasses.TranscriptRecordClass attribute gene_variants_missense_common gene 3 results download GeneRecordClasses.GeneRecordClass.uniprot_id http://edamontology.org/topic_3345 Data identity and mapping GeneRecordClasses.GeneRecordClass attribute uniprot_id gene GeneRecordClasses.GeneRecordClass.uniprot_id_internal http://edamontology.org/topic_3345 Data identity and mapping GeneRecordClasses.GeneRecordClass attribute uniprot_id_internal gene TranscriptRecordClasses.TranscriptRecordClass.uniprot_links http://edamontology.org/topic_3345 Data identity and mapping TranscriptRecordClasses.TranscriptRecordClass attribute uniprot_links transcript results From 1c3bd3bf52737d628f7269cc498053fff71042df Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Fri, 7 Aug 2026 13:14:30 -0400 Subject: [PATCH 84/98] Rename the genotype-ploidy attribute so it cannot be read as copy number "Ploidy (as called)" collided with the established meaning of ploidy in this model. apidb.ChrCopyNumber -> ChrCopyNumbers.ploidy is MEASURED chromosome copy number, surfaced by SequencesByPloidy under the display name "Copy Number/Ploidy", and GenesByCopyNumber's own description defines gene dose as ploidy * haploid_number. A reader seeing "Ploidy (as called) 1.01" on the gene page would reasonably take it as a copy-number statement about the gene. It is not. The attribute is total_ploidy_count / called_strain_count on apidb.VariationFeature - how many allele copies per sample the variant caller reported. Data provenance, not biology. Now "Genotype Ploidy (variant caller)", with help text that disclaims the copy-number reading outright and points at the CNV searches for the real thing. The help also notes that a haploid organism processed with diploid settings reads close to 2, so the afum case (2.01 for a haploid fungus) presents as a finding about the upstream calling rather than looking like a bug in this column. An XML comment records why the name is what it is, so it does not get "simplified" back to Ploidy. Only the record XML changed - the ontology rows carry a blank displayName column on purpose, so wb model was sufficient and individuals.txt is untouched. Verified on jbrestel.plasmodb.org after wb model: /service/record-types/gene reports the new displayName, the help text carries the disclaimer, the Sample Basis subsection renders "Genotype Ploidy (variant caller) 1.01", and the old label is gone from the page. Build clean, all logs silent including both error logs. Co-Authored-By: Claude Opus 5 --- Model/lib/wdk/model/records/geneRecord.xml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/Model/lib/wdk/model/records/geneRecord.xml b/Model/lib/wdk/model/records/geneRecord.xml index 9a506df7f2..01366d05fa 100644 --- a/Model/lib/wdk/model/records/geneRecord.xml +++ b/Model/lib/wdk/model/records/geneRecord.xml @@ -419,8 +419,15 @@ - + + From f3f46c606474b299c7494f6a8314c00b9ac34a58 Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Fri, 7 Aug 2026 17:12:19 -0400 Subject: [PATCH 85/98] Offer the reference strain in the HSSS sample filters The EDA dnaseq study holds only the resequenced isolates, so the reference strain never appeared in variation_sample_meta - even though HSSS has always carried it as strain id 1 (3D7 on PlasmoDB, TREU927 on TriTrypDB, Af293 on FungiDB). The searches could always include it; only the UI could not offer it. SamplesMetadataByStudyWithRef unions three synthesized attribute rows onto the attributevalue select: strain, organism, and dataset. Those are the identity a reference strain genuinely has - it has no collection site, host, or alignment statistics, and an unset attribute correctly excludes it from those facets. Keyed on provider_label rather than the VAR_ stable_id, because the label is site-specific (parasite_strain/parasite_organism vs fungal_strain/ fungus_organism - note fungus, not fungal). Values reuse the isolates' shape, so the reference joins the existing facet buckets instead of creating singletons: the organism facet stays one bucket at 217, not two. variation_sample_meta, _a and _b move to the new query; cnv_sample_meta keeps the original. The reference is real for HSSS but not for copy number, whose searches filter a table with no reference rows - offering it there would be an option that silently returns nothing. Verified on the running instance: all five HSSS filters report 217 samples and include the reference, CNV reports 216 and excludes it, and a search over 3D7 + B082 returns variants. Co-Authored-By: Claude Opus 5 --- .../questions/params/variationParams.xml | 123 +++++++++++++++++- ...-genetic-variation-searches-port-design.md | 67 ++++++++++ 2 files changed, 184 insertions(+), 6 deletions(-) diff --git a/Model/lib/wdk/model/questions/params/variationParams.xml b/Model/lib/wdk/model/questions/params/variationParams.xml index 374f4ed747..aac33710e0 100644 --- a/Model/lib/wdk/model/questions/params/variationParams.xml +++ b/Model/lib/wdk/model/questions/params/variationParams.xml @@ -55,6 +55,13 @@ strain names in HSSS's strainIdToName.dat - no mapping layer needed. The plugin writes them to a strains file and passes strains_are_names = 1. + One internal is NOT an EDA sample stable ID: the reference strain, which HSSS + carries as strain id 1 but EDA's dnaseq study omits, is synthesized by + SamplesMetadataByStudyWithRef from apidb.organism.strain_abbrev. That value is + chosen precisely because it is what strainIdToName.dat calls the strain, so the + invariant the paragraph above states - internal is a strain name HSSS knows - + still holds. See that query for the derivation. + minSelectedCount=2 as the snp original: polymorphism within a group of one is meaningless. @@ -63,8 +70,8 @@ the converse, and declaring it keeps the filter visibly downstream of the organism choice it is in fact scoped by. --> @@ -256,8 +271,8 @@ @@ -433,6 +448,102 @@ + + + + + + + + + + + + + + + which is exactly what this returns. + + The one place EDA is NOT taken verbatim is the dataset attribute. Its values + are EDA dataset stable IDs (DS_1d17c1883c), which is the right thing to store + but an opaque thing to show: the Dataset facet listed four DS_ hashes. The + LEFT JOIN swaps in apidbtuning.datasetpresenter.display_name. COALESCE keeps + the raw ID rather than dropping the sample if a presenter is missing, which on + a gated dev instance is a real possibility - datasetpresenter here holds only + the loaded datasets. + + The join is narrowed by provider_label so it can only ever fire on the dataset + attribute. Matching on the DS_ shape alone would work today (no other + attribute's value looks like a dataset ID) but would be a silent coincidence + to depend on. dataset_presenter_id is unique, so the join cannot multiply rows. + + Nothing downstream reads these strings - the filter passes `internal` (sample + IDs) to the searches - so changing the displayed value is display-only. --> @@ -440,10 +456,16 @@ @@ -505,10 +527,16 @@ _primary_genome_RSRC` | `DS_1d17c1883c` | `DS_071f05cd56` | +| `dataset_id` | `apidbtuning.datasetpresenter.display_name` for `_primary_genome_RSRC` | `Genome Sequence and Annotation` | `Genome Sequence and Annotation` | `internal` is `strain_abbrev` rather than the taxon name because that is what `strainIdToName.dat` calls the strain, and the plugin passes internals straight through as @@ -428,6 +428,34 @@ queries rather than one flag, because the difference is about which data exists. `GenesByNgsSnps` inherits the change for free — it uses `variation_sample_meta` (§4.5). +### 5.5 Showing dataset display names instead of `DS_` IDs (addendum, 2026-08-07) + +EDA stores the dataset attribute as a dataset stable ID (`DS_1d17c1883c`), which is the +right thing to store and an opaque thing to show — the Dataset facet listed four `DS_` +hashes. Both metadata queries now `LEFT JOIN apidbtuning.datasetpresenter` and select +`COALESCE(dp.display_name, av.string_value)`. + +Applied to **`SamplesMetadataByStudy` as well as `SamplesMetadataByStudyWithRef`**, so +`cnv_sample_meta` gets it too. The §5.4 fork is about which *samples* exist, not about +presentation; the two queries should differ in exactly one respect, and a facet that +reads `DS_302d3e3bc0` in the CNV searches and a sentence in the variation searches — for +the same samples — would be a worse bug than the one being fixed. + +Details that are load-bearing: + +- **`COALESCE`, not a plain join value** — keep the raw ID rather than nulling the + attribute when no presenter row exists. On a gated dev instance `datasetpresenter` + holds only the loaded datasets, so a miss is a real possibility. +- **The join is narrowed by `provider_label = '["dataset_id"]'`**, not left to match on + the `DS_` shape. Shape-matching works today because no other attribute's value looks + like a dataset ID, but that is a coincidence, not a constraint. +- **No row multiplication** — `dataset_presenter_id` is unique (57/57 distinct here). + Verified: row and sample counts are identical before and after (3771/216 and 3774/217). +- **Display-only** — the filter passes `internal` (sample stable IDs) to the searches; + nothing downstream reads these strings. Note this does mean a saved strategy's stored + filter value now carries the display name, which is why the change belongs in both + queries at once rather than being rolled out one filter at a time. + ## 6. Questions and categorization ### 6.1 `GenesByNgsSnps` (`geneQuestions.xml`) From c8a99bdd128fb53191734a9ab6477d1eb5ed21bb Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Fri, 7 Aug 2026 20:40:10 -0400 Subject: [PATCH 87/98] Rename the Variation record to Variant; display as "Short Variant" The record class was named Variation while every source_id it holds reads Variant__, and its own attributes already said Variant Type / Variant Call Set. "Variation" is the phenomenon; a record is one concrete allele at a locus, so it is a variant. Renames the WDK layer only - record class, question set and its five searches, query sets, param set, category ontology nodes, and the six model files - and sets displayName to Short Variant / Short Variants, naming what the record actually holds (SNVs and small indels) rather than variation in general. Deliberately unchanged: - urlName stays "variation" so existing record urls keep resolving; the feature is not public yet, so this can move in a separate change. - variation_sample_meta{,_a,_b} and chromosomeOptionalForVariations are contracts with the HSSS plugins in ApiCommonWebService; renaming them breaks the searches at run time, not build time. - apidb.Variation* and apidbtuning.VariationAttributes / GeneVariationSummary are database objects, out of the model's reach. Verified on a dev instance after wb ontology: all five searches resolve with expandParams, and the category tree keeps all 59 nodes in their sections with no stale VariationRecordClass references. Co-Authored-By: Claude Opus 5 --- Model/lib/wdk/apiCommonModel.xml | 14 +- .../model/questions/params/organismParams.xml | 4 +- ...{variationParams.xml => variantParams.xml} | 80 +++++------ .../model/questions/queries/geneQueries.xml | 30 ++--- .../questions/queries/genomicQueries.xml | 4 +- ...ariationQueries.xml => variantQueries.xml} | 102 +++++++------- ...tionQuestions.xml => variantQuestions.xml} | 66 +++++----- ...ueries.xml => variantAttributeQueries.xml} | 10 +- ...ariationRecords.xml => variantRecords.xml} | 29 ++-- ...bleQueries.xml => variantTableQueries.xml} | 2 +- Model/lib/wdk/ontology/individuals.txt | 124 +++++++++--------- 11 files changed, 236 insertions(+), 229 deletions(-) rename Model/lib/wdk/model/questions/params/{variationParams.xml => variantParams.xml} (92%) rename Model/lib/wdk/model/questions/queries/{variationQueries.xml => variantQueries.xml} (71%) rename Model/lib/wdk/model/questions/{variationQuestions.xml => variantQuestions.xml} (90%) rename Model/lib/wdk/model/records/{variationAttributeQueries.xml => variantAttributeQueries.xml} (96%) rename Model/lib/wdk/model/records/{variationRecords.xml => variantRecords.xml} (93%) rename Model/lib/wdk/model/records/{variationTableQueries.xml => variantTableQueries.xml} (97%) diff --git a/Model/lib/wdk/apiCommonModel.xml b/Model/lib/wdk/apiCommonModel.xml index 8a0b760fe3..2c592ee45b 100644 --- a/Model/lib/wdk/apiCommonModel.xml +++ b/Model/lib/wdk/apiCommonModel.xml @@ -476,13 +476,13 @@ Note that changing the sample subset will reset any group assignments you have a --> - - - - - - - + + + + + + + - + diff --git a/Model/lib/wdk/model/questions/params/variationParams.xml b/Model/lib/wdk/model/questions/params/variantParams.xml similarity index 92% rename from Model/lib/wdk/model/questions/params/variationParams.xml rename to Model/lib/wdk/model/questions/params/variantParams.xml index 9c30ab5718..7c6497c1b4 100644 --- a/Model/lib/wdk/model/questions/params/variationParams.xml +++ b/Model/lib/wdk/model/questions/params/variantParams.xml @@ -1,25 +1,25 @@ - - + - - Input a comma delimited set of Variation IDs, or upload a file + + Input a comma delimited set of Short Variant IDs, or upload a file + default once their short variant data loads. Omitted rather than invented. --> @@ -33,9 +33,9 @@ quote="false" because it is interpolated into an identifier, not compared as a string. visible="false" because it is derived from the organism, not chosen. The interpolation is safe: the value can only ever be one of the - rows VariationVQ.EdaSampleTableSuffix returns, never free user text. --> + rows VariantVQ.EdaSampleTableSuffix returns, never free user text. --> + dependedParamRef="organismParams.organismSinglePick,variantParams.eda_sample_table_suffix"> Select a set of samples whose genomic sequences will be compared. Use the sample characteristics to narrow the group, or accept all samples for the @@ -96,25 +100,25 @@ Unlike variation_sample_meta, this name is NOT a plugin contract - the CNV searches are plain sqlQueries with no plugin behind them. It is a separate param anyway, - because the minSelectedCount differs and because a param named for variation should + because the minSelectedCount differs and because a param named for short variants should not be what a genomic-sequence ploidy search depends on. Also deliberately points at SamplesMetadataByStudy, NOT the ...WithRef variant - the three variation filters use. The reference-strain row set is real for HSSS, + the three short variant filters use. The reference-strain row set is real for HSSS, which searches it as strain id 1, but not for copy number: the CNV searches filter c.eda_sample_stable_id against a CNV table that has no reference rows (genomicQueries.xml, geneQueries.xml), so offering it here would be an option that silently returns nothing. Two queries rather than one flag because the difference is about which data exists, not about presentation. - This lives in variationParams.xml rather than a neutral file by decision, not + This lives in variantParams.xml rather than a neutral file by decision, not oversight: see docs/superpowers/specs/2026-08-06-genetic-variation-searches-port-design.md s9.1. --> + dependedParamRef="organismParams.organismSinglePick,variantParams.eda_sample_table_suffix"> Choose the resequenced strains or samples to examine. Use the sample characteristics to narrow the group, or accept all samples for the organism you chose. @@ -220,7 +224,7 @@ + dependedParamRef="organismParams.organismSinglePick,variantParams.eda_sample_table_suffix"> Select the first group of samples to compare. Use the sample characteristics to narrow the group, or accept all samples for the organism you chose. @@ -271,11 +275,11 @@ + dependedParamRef="organismParams.organismSinglePick,variantParams.eda_sample_table_suffix"> Select the second group of samples to compare. It must differ from Set A; comparing a group against itself returns nothing useful. @@ -375,10 +379,10 @@ - - + - + @@ -507,7 +511,7 @@ NOTE the BARE $$organismSinglePick$$ below, with no surrounding quotes. The param is quote="true", so WDK supplies the quotes. The two vocab queries in - this same querySet - EdaSampleTableSuffix and ChromosomeForVariations - write + this same querySet - EdaSampleTableSuffix and ChromosomeForVariants - write it as '$$organismSinglePick$$' and are correct, because a flatVocabParam's vocab query resolves its depended values on a different path that substitutes raw. This is a filter param's metadata query, which goes through @@ -516,7 +520,7 @@ error. The non-vocab queries elsewhere (genomicQueries.xml, geneQueries.xml) all use the bare form; match those, not the neighbours. --> - + @@ -589,7 +593,7 @@ what makes them the tree's internal nodes. 'integer' folds into 'number' because WDK has no integer type. --> - + @@ -633,7 +637,7 @@ The 'Choose chromosome' sentinel is deliberate: it gives "no chromosome picked" a real value rather than an empty param. --> - + diff --git a/Model/lib/wdk/model/questions/queries/geneQueries.xml b/Model/lib/wdk/model/questions/queries/geneQueries.xml index dced67a7c1..73b4e9fe09 100644 --- a/Model/lib/wdk/model/questions/queries/geneQueries.xml +++ b/Model/lib/wdk/model/questions/queries/geneQueries.xml @@ -2848,32 +2848,32 @@ The override is required, not cosmetic. Plain organismSinglePick falls back to organismVQ.withGenes, i.e. every annotated organism in the project. Choosing one - with no dnaseq study makes variationParams.eda_sample_table_suffix return zero + with no dnaseq study makes variantParams.eda_sample_table_suffix return zero rows, so the samples filter has nothing to render and the search breaks rather - than returning an empty result. withVariationsTree restricts to organisms having + than returning an empty result. withVariantsTree restricts to organisms having an isolates/Dna_Seq datasource. --> + queryRef="organismVQ.withVariantsTree"> The organism you choose will determine the samples from which you can identify SNPs. - + - + SNPs are defined here as sequence differences between the selected samples. If you want to include sequence differences between the selected samples and the reference genome, then also include the reference sample in your search. - - + + - - + + @@ -5506,14 +5506,14 @@ select distinct ta.gene_source_id - - + + @@ -5614,14 +5614,14 @@ select distinct ta.gene_source_id - - + + diff --git a/Model/lib/wdk/model/questions/queries/genomicQueries.xml b/Model/lib/wdk/model/questions/queries/genomicQueries.xml index e69909a783..5c69a908f9 100644 --- a/Model/lib/wdk/model/questions/queries/genomicQueries.xml +++ b/Model/lib/wdk/model/questions/queries/genomicQueries.xml @@ -313,8 +313,8 @@ eda_sample_table_suffix / cnv_sample_meta dependency chain stays intact. --> - - + + diff --git a/Model/lib/wdk/model/questions/queries/variationQueries.xml b/Model/lib/wdk/model/questions/queries/variantQueries.xml similarity index 71% rename from Model/lib/wdk/model/questions/queries/variationQueries.xml rename to Model/lib/wdk/model/questions/queries/variantQueries.xml index 60403985bb..764b6df94b 100644 --- a/Model/lib/wdk/model/questions/queries/variationQueries.xml +++ b/Model/lib/wdk/model/questions/queries/variantQueries.xml @@ -1,30 +1,30 @@ - - + - - + + - + - + queryRef="organismVQ.withVariantsTree"> The Organism defines the species identity of the samples and the genome against which each sample's variants were called. After choosing an Organism, the set of samples available for forming groups is limited to samples aligned @@ -53,12 +53,12 @@ choose, since variants are identified by aligning that sample's reads to this genome. - - - - - - + + + + + + @@ -69,9 +69,9 @@ - + - - + queryRef="organismVQ.withVariantsTree"> The Organism defines the species identity of the samples and the genome against which each sample's variants were called. - - + + - - - - - + + + + + @@ -110,9 +110,9 @@ - + - - + queryRef="organismVQ.withVariantsTree"> The Organism defines the species identity of the samples and the genome against which each sample's variants were called. - - - - - - - + + + + + @@ -150,7 +150,7 @@ - + - + queryRef="organismVQ.withVariantsTree"> The Organism defines the species identity of the samples and the genome against which each sample's variants were called. - - - - - - - - - - - - + + + + + + + + + + + + diff --git a/Model/lib/wdk/model/questions/variationQuestions.xml b/Model/lib/wdk/model/questions/variantQuestions.xml similarity index 90% rename from Model/lib/wdk/model/questions/variationQuestions.xml rename to Model/lib/wdk/model/questions/variantQuestions.xml index 1813f6fe9b..7f0c479a83 100644 --- a/Model/lib/wdk/model/questions/variationQuestions.xml +++ b/Model/lib/wdk/model/questions/variantQuestions.xml @@ -1,28 +1,28 @@ - - + - - Find variations by ID. + Find short variants by ID.
+ Find short variants by ID.

Either enter the ID list manually, or upload a file that contains the list. IDs can be delimited by a comma, a semi colon, or any white spaces. @@ -32,9 +32,9 @@ - + - - + queryRef="VariantsBy.VariantsByIsolateGroup" + recordClassRef="VariantRecordClasses.VariantRecordClass"> + summary="variant_location,gene_ids,variant_type,PercentMinorAlleles,PercentIsolateCalls,Phenotype"/> - + - + queryRef="VariantsBy.VariantsByLocation" + recordClassRef="VariantRecordClasses.VariantRecordClass"> + summary="variant_location,gene_ids,variant_type,PercentMinorAlleles,PercentIsolateCalls,Phenotype"/> - + - + queryRef="VariantsBy.VariantsByGeneIds" + recordClassRef="VariantRecordClasses.VariantRecordClass"> + summary="variant_location,gene_ids,variant_type,PercentMinorAlleles,PercentIsolateCalls,Phenotype"/> - + - + queryRef="VariantsBy.VariantsByTwoIsolateGroups" + recordClassRef="VariantRecordClasses.VariantRecordClass"> + summary="variant_location,gene_ids,variant_type,MajorAlleleA,MajorAllelePctA,IsTriallelicA,MajorProductA,MajorProductIsVariableA,MajorAlleleB,MajorAllelePctB,IsTriallelicB,MajorProductB,MajorProductIsVariableB"/> - @@ -24,7 +24,7 @@ - + @@ -41,10 +41,10 @@ - + @@ -95,7 +95,7 @@ - + diff --git a/Model/lib/wdk/model/records/variationRecords.xml b/Model/lib/wdk/model/records/variantRecords.xml similarity index 93% rename from Model/lib/wdk/model/records/variationRecords.xml rename to Model/lib/wdk/model/records/variantRecords.xml index 1af7c83bdd..e84038dd0b 100644 --- a/Model/lib/wdk/model/records/variationRecords.xml +++ b/Model/lib/wdk/model/records/variantRecords.xml @@ -1,10 +1,13 @@ - - + + Variant_Pf3D7_01_v3_100057 @@ -25,12 +28,12 @@ FungiDB - + source_id project_id - + - + @@ -83,7 +86,7 @@ - + @@ -134,7 +137,7 @@ for the per-class frequencies."/> - + @@ -205,7 +208,7 @@
Organism
$$organism$$
-
Location
$$variation_location$$
+
Location
$$variant_location$$
Variant Type
$$variant_type$$
Coding
$$is_coding$$
Reference Strain
$$reference_strain$$
@@ -240,17 +243,17 @@ + queryRef="VariantTables.TranscriptProducts"> @@ -282,7 +285,7 @@ travels with every row, and the 19% disagreement between callers stays legible instead of requiring the user to read two tables and diff them. -->
+ queryRef="VariantTables.PredictedEffects"> diff --git a/Model/lib/wdk/model/records/variationTableQueries.xml b/Model/lib/wdk/model/records/variantTableQueries.xml similarity index 97% rename from Model/lib/wdk/model/records/variationTableQueries.xml rename to Model/lib/wdk/model/records/variantTableQueries.xml index c52ba12de7..e84e6db1f5 100644 --- a/Model/lib/wdk/model/records/variationTableQueries.xml +++ b/Model/lib/wdk/model/records/variantTableQueries.xml @@ -1,6 +1,6 @@ - diff --git a/Model/lib/wdk/ontology/individuals.txt b/Model/lib/wdk/ontology/individuals.txt index b7e53b0d6c..6e5ffe6181 100644 --- a/Model/lib/wdk/ontology/individuals.txt +++ b/Model/lib/wdk/ontology/individuals.txt @@ -2,9 +2,9 @@ eupath/eupath.owl#recordClassName eupath/eupath.owl#targetType eupath/eupath.owl#name EUPATH_0000052 eupath/eupath.owl#shortDisplayName eupath/eupath.owl#description eupath/eupath.owl#geneOrTranscript EUPATH_0000274 eupath/eupath.owl#scope eupath/eupath.owl#scope eupath/eupath.owl#scope GenomicSequencePropertiesCategory GenomicSequencePropertiesCategory Genomic Sequence Properties 5 GenomicSequenceLocationCategory GenomicSequenceLocationCategory Genomic Location 6 -VariationSnpAlleleCategory http://edamontology.org/topic_2885 DNA polymorphism VariationSnpAlleleCategory SNP Alleles 1 -VariationIndelAlleleCategory http://edamontology.org/topic_2885 DNA polymorphism VariationIndelAlleleCategory Indel Alleles 2 -VariationStrainStatsCategory http://edamontology.org/topic_2885 DNA polymorphism VariationStrainStatsCategory Strain Statistics 3 +VariantSnpAlleleCategory http://edamontology.org/topic_2885 DNA polymorphism VariantSnpAlleleCategory SNP Alleles 1 +VariantIndelAlleleCategory http://edamontology.org/topic_2885 DNA polymorphism VariantIndelAlleleCategory Indel Alleles 2 +VariantStrainStatsCategory http://edamontology.org/topic_2885 DNA polymorphism VariantStrainStatsCategory Strain Statistics 3 TextCategory TextCategory Text 1 AlignmentsCategory http://edamontology.org/topic_0080 Sequence Analysis AlignmentsCategory BLAT and Blast Alignments CodingPotentialCategory http://edamontology.org/topic_0080 Sequence Analysis CodingPotentialCategory Coding Potential @@ -1148,62 +1148,62 @@ JbrowseRecordClasses.JbrowseGeneRecordClass.location_text GenomicSequenceLocatio JbrowseRecordClasses.JbrowseGeneRecordClass.GOTerms http://edamontology.org/topic_1775 Function analysis JbrowseRecordClasses.JbrowseGeneRecordClass table GOTerms transcript record JbrowseRecordClasses.Jbrowse.GeneRecordClass.GeneTranscripts http://edamontology.org/topic_0114 Gene Structure JbrowseRecordClasses.JbrowseGeneRecordClass table GeneTranscripts gene record -VariationRecordClasses.VariationRecordClass.variation_location GenomicSequenceLocationCategory GenomicSequenceLocationCategory VariationRecordClasses.VariationRecordClass attribute variation_location results record download -VariationRecordClasses.VariationRecordClass.sequence_source_id GenomicSequenceLocationCategory GenomicSequenceLocationCategory VariationRecordClasses.VariationRecordClass attribute sequence_source_id results record download -VariationRecordClasses.VariationRecordClass.location GenomicSequenceLocationCategory GenomicSequenceLocationCategory VariationRecordClasses.VariationRecordClass attribute location results record download -VariationRecordClasses.VariationRecordClass.location_text GenomicSequenceLocationCategory GenomicSequenceLocationCategory VariationRecordClasses.VariationRecordClass attribute location_text record-internal -VariationRecordClasses.VariationRecordClass.chromosome_order_num GenomicSequenceLocationCategory GenomicSequenceLocationCategory VariationRecordClasses.VariationRecordClass attribute chromosome_order_num record-internal -VariationRecordClasses.VariationRecordClass.organism GenomicSequenceLocationCategory GenomicSequenceLocationCategory VariationRecordClasses.VariationRecordClass attribute organism results record download -VariationRecordClasses.VariationRecordClass.organism_text GenomicSequenceLocationCategory GenomicSequenceLocationCategory VariationRecordClasses.VariationRecordClass attribute organism_text record-internal -VariationRecordClasses.VariationRecordClass.formatted_organism GenomicSequenceLocationCategory GenomicSequenceLocationCategory VariationRecordClasses.VariationRecordClass attribute formatted_organism record-internal -VariationRecordClasses.VariationRecordClass.ncbi_tax_id GenomicSequenceLocationCategory GenomicSequenceLocationCategory VariationRecordClasses.VariationRecordClass attribute ncbi_tax_id record download -VariationRecordClasses.VariationRecordClass.dataset GenomicSequenceLocationCategory GenomicSequenceLocationCategory VariationRecordClasses.VariationRecordClass attribute dataset results record download -VariationRecordClasses.VariationRecordClass.variant_type http://edamontology.org/topic_2885 DNA Polymorphism VariationRecordClasses.VariationRecordClass attribute variant_type results record download -VariationRecordClasses.VariationRecordClass.is_coding http://edamontology.org/topic_2885 DNA Polymorphism VariationRecordClasses.VariationRecordClass attribute is_coding results record download -VariationRecordClasses.VariationRecordClass.reference_strain http://edamontology.org/topic_2885 DNA Polymorphism VariationRecordClasses.VariationRecordClass attribute reference_strain results record download -VariationRecordClasses.VariationRecordClass.record_overview http://edamontology.org/topic_0219 annot and curation VariationRecordClasses.VariationRecordClass attribute record_overview record-internal -VariationRecordClasses.VariationRecordClass.snp_ref_allele VariationSnpAlleleCategory SNP Alleles VariationRecordClasses.VariationRecordClass attribute snp_ref_allele 1 results record download -VariationRecordClasses.VariationRecordClass.snp_major_allele VariationSnpAlleleCategory SNP Alleles VariationRecordClasses.VariationRecordClass attribute snp_major_allele 2 results record download -VariationRecordClasses.VariationRecordClass.snp_major_allele_frequency VariationSnpAlleleCategory SNP Alleles VariationRecordClasses.VariationRecordClass attribute snp_major_allele_frequency 3 results record download -VariationRecordClasses.VariationRecordClass.snp_major_allele_strain_count VariationSnpAlleleCategory SNP Alleles VariationRecordClasses.VariationRecordClass attribute snp_major_allele_strain_count 4 results record download -VariationRecordClasses.VariationRecordClass.snp_minor_allele VariationSnpAlleleCategory SNP Alleles VariationRecordClasses.VariationRecordClass attribute snp_minor_allele 6 results record download -VariationRecordClasses.VariationRecordClass.snp_minor_allele_frequency VariationSnpAlleleCategory SNP Alleles VariationRecordClasses.VariationRecordClass attribute snp_minor_allele_frequency 7 results record download -VariationRecordClasses.VariationRecordClass.snp_minor_allele_strain_count VariationSnpAlleleCategory SNP Alleles VariationRecordClasses.VariationRecordClass attribute snp_minor_allele_strain_count 8 results record download -VariationRecordClasses.VariationRecordClass.snp_major_genomic_hgvs VariationSnpAlleleCategory SNP Alleles VariationRecordClasses.VariationRecordClass attribute snp_major_genomic_hgvs 10 results record download -VariationRecordClasses.VariationRecordClass.snp_minor_genomic_hgvs VariationSnpAlleleCategory SNP Alleles VariationRecordClasses.VariationRecordClass attribute snp_minor_genomic_hgvs 11 results record download -VariationRecordClasses.VariationRecordClass.snp_major_allele_and_freq VariationSnpAlleleCategory SNP Alleles VariationRecordClasses.VariationRecordClass attribute snp_major_allele_and_freq 5 record download -VariationRecordClasses.VariationRecordClass.snp_minor_allele_and_freq VariationSnpAlleleCategory SNP Alleles VariationRecordClasses.VariationRecordClass attribute snp_minor_allele_and_freq 9 record download -VariationRecordClasses.VariationRecordClass.indel_ref_allele VariationIndelAlleleCategory Indel Alleles VariationRecordClasses.VariationRecordClass attribute indel_ref_allele 1 results record download -VariationRecordClasses.VariationRecordClass.indel_major_allele VariationIndelAlleleCategory Indel Alleles VariationRecordClasses.VariationRecordClass attribute indel_major_allele 2 results record download -VariationRecordClasses.VariationRecordClass.indel_major_allele_frequency VariationIndelAlleleCategory Indel Alleles VariationRecordClasses.VariationRecordClass attribute indel_major_allele_frequency 3 results record download -VariationRecordClasses.VariationRecordClass.indel_major_allele_strain_count VariationIndelAlleleCategory Indel Alleles VariationRecordClasses.VariationRecordClass attribute indel_major_allele_strain_count 4 results record download -VariationRecordClasses.VariationRecordClass.indel_minor_allele VariationIndelAlleleCategory Indel Alleles VariationRecordClasses.VariationRecordClass attribute indel_minor_allele 6 results record download -VariationRecordClasses.VariationRecordClass.indel_minor_allele_frequency VariationIndelAlleleCategory Indel Alleles VariationRecordClasses.VariationRecordClass attribute indel_minor_allele_frequency 7 results record download -VariationRecordClasses.VariationRecordClass.indel_minor_allele_strain_count VariationIndelAlleleCategory Indel Alleles VariationRecordClasses.VariationRecordClass attribute indel_minor_allele_strain_count 8 results record download -VariationRecordClasses.VariationRecordClass.indel_major_genomic_hgvs VariationIndelAlleleCategory Indel Alleles VariationRecordClasses.VariationRecordClass attribute indel_major_genomic_hgvs 10 results record download -VariationRecordClasses.VariationRecordClass.indel_minor_genomic_hgvs VariationIndelAlleleCategory Indel Alleles VariationRecordClasses.VariationRecordClass attribute indel_minor_genomic_hgvs 11 results record download -VariationRecordClasses.VariationRecordClass.indel_major_allele_and_freq VariationIndelAlleleCategory Indel Alleles VariationRecordClasses.VariationRecordClass attribute indel_major_allele_and_freq 5 record download -VariationRecordClasses.VariationRecordClass.indel_minor_allele_and_freq VariationIndelAlleleCategory Indel Alleles VariationRecordClasses.VariationRecordClass attribute indel_minor_allele_and_freq 9 record download -VariationRecordClasses.VariationRecordClass.indel_frame_effect VariationIndelAlleleCategory Indel Alleles VariationRecordClasses.VariationRecordClass attribute indel_frame_effect 12 results record download -VariationRecordClasses.VariationRecordClass.distinct_strain_count VariationStrainStatsCategory Strain Statistics VariationRecordClasses.VariationRecordClass attribute distinct_strain_count 1 results record download -VariationRecordClasses.VariationRecordClass.called_strain_count VariationStrainStatsCategory Strain Statistics VariationRecordClasses.VariationRecordClass attribute called_strain_count 2 results record download -VariationRecordClasses.VariationRecordClass.no_call_strain_count VariationStrainStatsCategory Strain Statistics VariationRecordClasses.VariationRecordClass attribute no_call_strain_count 3 results record download -VariationRecordClasses.VariationRecordClass.call_rate VariationStrainStatsCategory Strain Statistics VariationRecordClasses.VariationRecordClass attribute call_rate 4 results record download -VariationRecordClasses.VariationRecordClass.total_ploidy_count VariationStrainStatsCategory Strain Statistics VariationRecordClasses.VariationRecordClass attribute total_ploidy_count 5 results record download -VariationRecordClasses.VariationRecordClass.het_strain_count VariationStrainStatsCategory Strain Statistics VariationRecordClasses.VariationRecordClass attribute het_strain_count 6 results record download -VariationRecordClasses.VariationRecordClass.ref_allele_frequency VariationStrainStatsCategory Strain Statistics VariationRecordClasses.VariationRecordClass attribute ref_allele_frequency 7 results record download -VariationRecordClasses.VariationRecordClass.gene_ids http://edamontology.org/topic_0199 Genetic Variation VariationRecordClasses.VariationRecordClass attribute gene_ids results record download -VariationRecordClasses.VariationRecordClass.most_severe_impact_snpeff http://edamontology.org/topic_0199 Genetic Variation VariationRecordClasses.VariationRecordClass attribute most_severe_impact_snpeff results record download -VariationRecordClasses.VariationRecordClass.most_severe_impact_product_call http://edamontology.org/topic_0199 Genetic Variation VariationRecordClasses.VariationRecordClass attribute most_severe_impact_product_call results record download -VariationRecordClasses.VariationRecordClass.effect_summary_snpeff http://edamontology.org/topic_0199 Genetic Variation VariationRecordClasses.VariationRecordClass attribute effect_summary_snpeff results record download -VariationRecordClasses.VariationRecordClass.effect_summary_product_call http://edamontology.org/topic_0199 Genetic Variation VariationRecordClasses.VariationRecordClass attribute effect_summary_product_call results record download -VariationRecordClasses.VariationRecordClass.collapsed_allele http://edamontology.org/topic_0199 Genetic Variation VariationRecordClasses.VariationRecordClass attribute collapsed_allele results record download -VariationRecordClasses.VariationRecordClass.collapsed_minor_allele_frequency http://edamontology.org/topic_0199 Genetic Variation VariationRecordClasses.VariationRecordClass attribute collapsed_minor_allele_frequency results record download -VariationRecordClasses.VariationRecordClass.gene_count http://edamontology.org/topic_0199 Genetic Variation VariationRecordClasses.VariationRecordClass attribute gene_count record-internal -VariationRecordClasses.VariationRecordClass.TranscriptProducts http://edamontology.org/topic_0199 Genetic Variation VariationRecordClasses.VariationRecordClass table TranscriptProducts record download -VariationRecordClasses.VariationRecordClass.PredictedEffects http://edamontology.org/topic_0199 Genetic Variation VariationRecordClasses.VariationRecordClass table PredictedEffects record download -VariationRecordClasses.VariationRecordClass.VariationQuestions.VariationBySourceId http://edamontology.org/topic_0199 Genetic Variation VariationRecordClasses.VariationRecordClass search VariationQuestions.VariationBySourceId menu webservice -VariationRecordClasses.VariationRecordClass.VariationQuestions.VariationsByIsolateGroup http://edamontology.org/topic_0199 Genetic Variation VariationRecordClasses.VariationRecordClass search VariationQuestions.VariationsByIsolateGroup menu webservice -VariationRecordClasses.VariationRecordClass.VariationQuestions.VariationsByLocation http://edamontology.org/topic_0199 Genetic Variation VariationRecordClasses.VariationRecordClass search VariationQuestions.VariationsByLocation menu webservice -VariationRecordClasses.VariationRecordClass.VariationQuestions.VariationsByGeneIds http://edamontology.org/topic_0199 Genetic Variation VariationRecordClasses.VariationRecordClass search VariationQuestions.VariationsByGeneIds menu webservice -VariationRecordClasses.VariationRecordClass.VariationQuestions.VariationsByTwoIsolateGroups http://edamontology.org/topic_0199 Genetic Variation VariationRecordClasses.VariationRecordClass search VariationQuestions.VariationsByTwoIsolateGroups menu webservice +VariantRecordClasses.VariantRecordClass.variant_location GenomicSequenceLocationCategory GenomicSequenceLocationCategory VariantRecordClasses.VariantRecordClass attribute variant_location results record download +VariantRecordClasses.VariantRecordClass.sequence_source_id GenomicSequenceLocationCategory GenomicSequenceLocationCategory VariantRecordClasses.VariantRecordClass attribute sequence_source_id results record download +VariantRecordClasses.VariantRecordClass.location GenomicSequenceLocationCategory GenomicSequenceLocationCategory VariantRecordClasses.VariantRecordClass attribute location results record download +VariantRecordClasses.VariantRecordClass.location_text GenomicSequenceLocationCategory GenomicSequenceLocationCategory VariantRecordClasses.VariantRecordClass attribute location_text record-internal +VariantRecordClasses.VariantRecordClass.chromosome_order_num GenomicSequenceLocationCategory GenomicSequenceLocationCategory VariantRecordClasses.VariantRecordClass attribute chromosome_order_num record-internal +VariantRecordClasses.VariantRecordClass.organism GenomicSequenceLocationCategory GenomicSequenceLocationCategory VariantRecordClasses.VariantRecordClass attribute organism results record download +VariantRecordClasses.VariantRecordClass.organism_text GenomicSequenceLocationCategory GenomicSequenceLocationCategory VariantRecordClasses.VariantRecordClass attribute organism_text record-internal +VariantRecordClasses.VariantRecordClass.formatted_organism GenomicSequenceLocationCategory GenomicSequenceLocationCategory VariantRecordClasses.VariantRecordClass attribute formatted_organism record-internal +VariantRecordClasses.VariantRecordClass.ncbi_tax_id GenomicSequenceLocationCategory GenomicSequenceLocationCategory VariantRecordClasses.VariantRecordClass attribute ncbi_tax_id record download +VariantRecordClasses.VariantRecordClass.dataset GenomicSequenceLocationCategory GenomicSequenceLocationCategory VariantRecordClasses.VariantRecordClass attribute dataset results record download +VariantRecordClasses.VariantRecordClass.variant_type http://edamontology.org/topic_2885 DNA Polymorphism VariantRecordClasses.VariantRecordClass attribute variant_type results record download +VariantRecordClasses.VariantRecordClass.is_coding http://edamontology.org/topic_2885 DNA Polymorphism VariantRecordClasses.VariantRecordClass attribute is_coding results record download +VariantRecordClasses.VariantRecordClass.reference_strain http://edamontology.org/topic_2885 DNA Polymorphism VariantRecordClasses.VariantRecordClass attribute reference_strain results record download +VariantRecordClasses.VariantRecordClass.record_overview http://edamontology.org/topic_0219 annot and curation VariantRecordClasses.VariantRecordClass attribute record_overview record-internal +VariantRecordClasses.VariantRecordClass.snp_ref_allele VariantSnpAlleleCategory SNP Alleles VariantRecordClasses.VariantRecordClass attribute snp_ref_allele 1 results record download +VariantRecordClasses.VariantRecordClass.snp_major_allele VariantSnpAlleleCategory SNP Alleles VariantRecordClasses.VariantRecordClass attribute snp_major_allele 2 results record download +VariantRecordClasses.VariantRecordClass.snp_major_allele_frequency VariantSnpAlleleCategory SNP Alleles VariantRecordClasses.VariantRecordClass attribute snp_major_allele_frequency 3 results record download +VariantRecordClasses.VariantRecordClass.snp_major_allele_strain_count VariantSnpAlleleCategory SNP Alleles VariantRecordClasses.VariantRecordClass attribute snp_major_allele_strain_count 4 results record download +VariantRecordClasses.VariantRecordClass.snp_minor_allele VariantSnpAlleleCategory SNP Alleles VariantRecordClasses.VariantRecordClass attribute snp_minor_allele 6 results record download +VariantRecordClasses.VariantRecordClass.snp_minor_allele_frequency VariantSnpAlleleCategory SNP Alleles VariantRecordClasses.VariantRecordClass attribute snp_minor_allele_frequency 7 results record download +VariantRecordClasses.VariantRecordClass.snp_minor_allele_strain_count VariantSnpAlleleCategory SNP Alleles VariantRecordClasses.VariantRecordClass attribute snp_minor_allele_strain_count 8 results record download +VariantRecordClasses.VariantRecordClass.snp_major_genomic_hgvs VariantSnpAlleleCategory SNP Alleles VariantRecordClasses.VariantRecordClass attribute snp_major_genomic_hgvs 10 results record download +VariantRecordClasses.VariantRecordClass.snp_minor_genomic_hgvs VariantSnpAlleleCategory SNP Alleles VariantRecordClasses.VariantRecordClass attribute snp_minor_genomic_hgvs 11 results record download +VariantRecordClasses.VariantRecordClass.snp_major_allele_and_freq VariantSnpAlleleCategory SNP Alleles VariantRecordClasses.VariantRecordClass attribute snp_major_allele_and_freq 5 record download +VariantRecordClasses.VariantRecordClass.snp_minor_allele_and_freq VariantSnpAlleleCategory SNP Alleles VariantRecordClasses.VariantRecordClass attribute snp_minor_allele_and_freq 9 record download +VariantRecordClasses.VariantRecordClass.indel_ref_allele VariantIndelAlleleCategory Indel Alleles VariantRecordClasses.VariantRecordClass attribute indel_ref_allele 1 results record download +VariantRecordClasses.VariantRecordClass.indel_major_allele VariantIndelAlleleCategory Indel Alleles VariantRecordClasses.VariantRecordClass attribute indel_major_allele 2 results record download +VariantRecordClasses.VariantRecordClass.indel_major_allele_frequency VariantIndelAlleleCategory Indel Alleles VariantRecordClasses.VariantRecordClass attribute indel_major_allele_frequency 3 results record download +VariantRecordClasses.VariantRecordClass.indel_major_allele_strain_count VariantIndelAlleleCategory Indel Alleles VariantRecordClasses.VariantRecordClass attribute indel_major_allele_strain_count 4 results record download +VariantRecordClasses.VariantRecordClass.indel_minor_allele VariantIndelAlleleCategory Indel Alleles VariantRecordClasses.VariantRecordClass attribute indel_minor_allele 6 results record download +VariantRecordClasses.VariantRecordClass.indel_minor_allele_frequency VariantIndelAlleleCategory Indel Alleles VariantRecordClasses.VariantRecordClass attribute indel_minor_allele_frequency 7 results record download +VariantRecordClasses.VariantRecordClass.indel_minor_allele_strain_count VariantIndelAlleleCategory Indel Alleles VariantRecordClasses.VariantRecordClass attribute indel_minor_allele_strain_count 8 results record download +VariantRecordClasses.VariantRecordClass.indel_major_genomic_hgvs VariantIndelAlleleCategory Indel Alleles VariantRecordClasses.VariantRecordClass attribute indel_major_genomic_hgvs 10 results record download +VariantRecordClasses.VariantRecordClass.indel_minor_genomic_hgvs VariantIndelAlleleCategory Indel Alleles VariantRecordClasses.VariantRecordClass attribute indel_minor_genomic_hgvs 11 results record download +VariantRecordClasses.VariantRecordClass.indel_major_allele_and_freq VariantIndelAlleleCategory Indel Alleles VariantRecordClasses.VariantRecordClass attribute indel_major_allele_and_freq 5 record download +VariantRecordClasses.VariantRecordClass.indel_minor_allele_and_freq VariantIndelAlleleCategory Indel Alleles VariantRecordClasses.VariantRecordClass attribute indel_minor_allele_and_freq 9 record download +VariantRecordClasses.VariantRecordClass.indel_frame_effect VariantIndelAlleleCategory Indel Alleles VariantRecordClasses.VariantRecordClass attribute indel_frame_effect 12 results record download +VariantRecordClasses.VariantRecordClass.distinct_strain_count VariantStrainStatsCategory Strain Statistics VariantRecordClasses.VariantRecordClass attribute distinct_strain_count 1 results record download +VariantRecordClasses.VariantRecordClass.called_strain_count VariantStrainStatsCategory Strain Statistics VariantRecordClasses.VariantRecordClass attribute called_strain_count 2 results record download +VariantRecordClasses.VariantRecordClass.no_call_strain_count VariantStrainStatsCategory Strain Statistics VariantRecordClasses.VariantRecordClass attribute no_call_strain_count 3 results record download +VariantRecordClasses.VariantRecordClass.call_rate VariantStrainStatsCategory Strain Statistics VariantRecordClasses.VariantRecordClass attribute call_rate 4 results record download +VariantRecordClasses.VariantRecordClass.total_ploidy_count VariantStrainStatsCategory Strain Statistics VariantRecordClasses.VariantRecordClass attribute total_ploidy_count 5 results record download +VariantRecordClasses.VariantRecordClass.het_strain_count VariantStrainStatsCategory Strain Statistics VariantRecordClasses.VariantRecordClass attribute het_strain_count 6 results record download +VariantRecordClasses.VariantRecordClass.ref_allele_frequency VariantStrainStatsCategory Strain Statistics VariantRecordClasses.VariantRecordClass attribute ref_allele_frequency 7 results record download +VariantRecordClasses.VariantRecordClass.gene_ids http://edamontology.org/topic_0199 Genetic Variation VariantRecordClasses.VariantRecordClass attribute gene_ids results record download +VariantRecordClasses.VariantRecordClass.most_severe_impact_snpeff http://edamontology.org/topic_0199 Genetic Variation VariantRecordClasses.VariantRecordClass attribute most_severe_impact_snpeff results record download +VariantRecordClasses.VariantRecordClass.most_severe_impact_product_call http://edamontology.org/topic_0199 Genetic Variation VariantRecordClasses.VariantRecordClass attribute most_severe_impact_product_call results record download +VariantRecordClasses.VariantRecordClass.effect_summary_snpeff http://edamontology.org/topic_0199 Genetic Variation VariantRecordClasses.VariantRecordClass attribute effect_summary_snpeff results record download +VariantRecordClasses.VariantRecordClass.effect_summary_product_call http://edamontology.org/topic_0199 Genetic Variation VariantRecordClasses.VariantRecordClass attribute effect_summary_product_call results record download +VariantRecordClasses.VariantRecordClass.collapsed_allele http://edamontology.org/topic_0199 Genetic Variation VariantRecordClasses.VariantRecordClass attribute collapsed_allele results record download +VariantRecordClasses.VariantRecordClass.collapsed_minor_allele_frequency http://edamontology.org/topic_0199 Genetic Variation VariantRecordClasses.VariantRecordClass attribute collapsed_minor_allele_frequency results record download +VariantRecordClasses.VariantRecordClass.gene_count http://edamontology.org/topic_0199 Genetic Variation VariantRecordClasses.VariantRecordClass attribute gene_count record-internal +VariantRecordClasses.VariantRecordClass.TranscriptProducts http://edamontology.org/topic_0199 Genetic Variation VariantRecordClasses.VariantRecordClass table TranscriptProducts record download +VariantRecordClasses.VariantRecordClass.PredictedEffects http://edamontology.org/topic_0199 Genetic Variation VariantRecordClasses.VariantRecordClass table PredictedEffects record download +VariantRecordClasses.VariantRecordClass.VariantQuestions.VariantBySourceId http://edamontology.org/topic_0199 Genetic Variation VariantRecordClasses.VariantRecordClass search VariantQuestions.VariantBySourceId menu webservice +VariantRecordClasses.VariantRecordClass.VariantQuestions.VariantsByIsolateGroup http://edamontology.org/topic_0199 Genetic Variation VariantRecordClasses.VariantRecordClass search VariantQuestions.VariantsByIsolateGroup menu webservice +VariantRecordClasses.VariantRecordClass.VariantQuestions.VariantsByLocation http://edamontology.org/topic_0199 Genetic Variation VariantRecordClasses.VariantRecordClass search VariantQuestions.VariantsByLocation menu webservice +VariantRecordClasses.VariantRecordClass.VariantQuestions.VariantsByGeneIds http://edamontology.org/topic_0199 Genetic Variation VariantRecordClasses.VariantRecordClass search VariantQuestions.VariantsByGeneIds menu webservice +VariantRecordClasses.VariantRecordClass.VariantQuestions.VariantsByTwoIsolateGroups http://edamontology.org/topic_0199 Genetic Variation VariantRecordClasses.VariantRecordClass search VariantQuestions.VariantsByTwoIsolateGroups menu webservice From 24935d7e5f3052013704eadced99fd4d90405951 Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Fri, 7 Aug 2026 20:54:27 -0400 Subject: [PATCH 88/98] Mark the four HSSS searches as SNV-only in their display names HSSS computes over single-nucleotide variants only, so these four cannot return an indel no matter what the record class is called. VariantBySourceId keeps its neutral name because an ID lookup goes to the tuning table and does return both. Prefixes shortDisplayName too: a strategy box showing bare "Location" or "Two Groups" is exactly where the SNV-only scope would otherwise be lost. Co-Authored-By: Claude Opus 5 --- .../wdk/model/questions/variantQuestions.xml | 31 ++++++++++++------- 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/Model/lib/wdk/model/questions/variantQuestions.xml b/Model/lib/wdk/model/questions/variantQuestions.xml index 7f0c479a83..716b344b71 100644 --- a/Model/lib/wdk/model/questions/variantQuestions.xml +++ b/Model/lib/wdk/model/questions/variantQuestions.xml @@ -44,14 +44,21 @@ samples throughout and the filter is built from EDA. The internal search name stays VariantsByIsolateGroup. + displayName also leads with SNV, as do the other three HSSS searches. HSSS + computes over single-nucleotide variants only, so these four cannot return + an indel however the record class is named. VariantBySourceId is not + prefixed, because an ID lookup goes to the tuning table and does return + both. The record is "Short Variant" precisely because it holds both; the + prefix marks the four searches that see only half of it. + No searchCategory: it groups searches within a set, and with two short variant searches there is nothing to group. It arrives with ByLocation/ByGeneIds. noSummaryOnSingleRecord is deliberately NOT set: unlike an ID lookup, a one-hit analytical result is a finding the user wants to see in context. --> @@ -60,7 +67,7 @@ @@ -133,8 +140,8 @@ @@ -143,7 +150,7 @@ @@ -217,8 +224,8 @@ @@ -227,7 +234,7 @@ @@ -299,8 +306,8 @@ @@ -312,7 +319,7 @@ From bfb84182b1bb42faa500582ec49a95d400ca7617 Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Fri, 7 Aug 2026 21:01:16 -0400 Subject: [PATCH 89/98] Name the region and gene SNV searches for what they compute "SNV Gene ID(s)" and "SNV Genomic Location" named their input; all four HSSS searches compute differences within a sample group, so they now say so: "SNV Differences for Gene(s)" and "SNV Differences for Genomic Location". Co-Authored-By: Claude Opus 5 --- Model/lib/wdk/model/questions/variantQuestions.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Model/lib/wdk/model/questions/variantQuestions.xml b/Model/lib/wdk/model/questions/variantQuestions.xml index 716b344b71..287fa45911 100644 --- a/Model/lib/wdk/model/questions/variantQuestions.xml +++ b/Model/lib/wdk/model/questions/variantQuestions.xml @@ -140,7 +140,7 @@ @@ -224,7 +224,7 @@ From 7312822a11f2ef331b62f52a51858716fa4a56e7 Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Fri, 7 Aug 2026 21:57:39 -0400 Subject: [PATCH 90/98] Rename the GenesByNgsSnps display name to SNV Characteristics Co-Authored-By: Claude Opus 5 --- Model/lib/wdk/model/questions/geneQuestions.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Model/lib/wdk/model/questions/geneQuestions.xml b/Model/lib/wdk/model/questions/geneQuestions.xml index 70402ca6a1..42549d66c0 100644 --- a/Model/lib/wdk/model/questions/geneQuestions.xml +++ b/Model/lib/wdk/model/questions/geneQuestions.xml @@ -1467,8 +1467,8 @@ Find genes which are differentially expressed in an RNA-Seq experiment based on Date: Fri, 7 Aug 2026 22:37:24 -0400 Subject: [PATCH 91/98] Add GenesByVariantCharacteristics, a precomputed SNV characteristics search The sibling of GenesByNgsSnps. That one runs HSSS over a sample set the user picks and takes minutes; this one reads apidbtuning.GeneVariationSummary, returns in milliseconds, and reports the same numbers the gene record page shows. Neither replaces the other: dropping the HSSS search would delete the sample-set comparison, and not having this one leaves the record page and the search disagreeing about the same gene with no explanation. GenesByNgsSnps is renamed to "SNV Characteristics Within a Group of Samples" to say so. Filtering is ONE filterParam over sixteen statistics, not sixteen range params. The range-param version was written first and abandoned: a numberRangeParam cannot express "untouched", so every range applies on every run and any gene NULL in that column is excluded - and GeneVariationSummary NULLs its frequency statistics below their allele floors on purpose, which for tbruTREU927 is 100% of rows for three columns. Working around that needed a full-span escape clause with bounds hardcoded in two files, which then broke when NumberRangeParamHandler rounded a value onto the declared ceiling. The filterParam has none of it: a gene with no value contributes no metadata row, distributions come from the data, and adding a statistic is one ontology row plus one VALUES tuple. Also adds organismVQ.withVariantSummary, scoping the organism tree to organisms that actually have a summary row rather than to a datasource type. Verified against unidb_shu_a on a dev instance: the impact distribution sums to all 5,579 pfal genes (HIGH 2,797, matching SQL), piN/piS reports 2,370 - exactly the genes with a non-null value - and filtered runs match direct SQL counts (piN/piS >= 1.5 -> 271 genes/272 transcripts; n_lof >= 5 -> 317/319). Co-Authored-By: Claude Opus 5 --- .../lib/wdk/model/questions/geneQuestions.xml | 118 +++++++++- .../wdk/model/questions/params/geneParams.xml | 213 ++++++++++++++++++ .../model/questions/params/organismParams.xml | 40 ++++ .../model/questions/queries/geneQueries.xml | 74 ++++++ Model/lib/wdk/ontology/individuals.txt | 1 + 5 files changed, 444 insertions(+), 2 deletions(-) diff --git a/Model/lib/wdk/model/questions/geneQuestions.xml b/Model/lib/wdk/model/questions/geneQuestions.xml index 42549d66c0..6d41b943fa 100644 --- a/Model/lib/wdk/model/questions/geneQuestions.xml +++ b/Model/lib/wdk/model/questions/geneQuestions.xml @@ -1467,8 +1467,8 @@ Find genes which are differentially expressed in an RNA-Seq experiment based on + + + + + + + + + + + + + + + +
  • This search does not take a sample set. If you want to compare a + group of samples you chose - or two groups against each other - use the + "SNV Characteristics Within a Group of Samples" search instead. That one + recomputes from the raw alignments and takes read-frequency and + percent-called thresholds; this one reads precomputed values and returns + immediately.
    • + +
  • Pick an organism first. The distributions in the filter are drawn from + that organism's genes, and variant density differs by an order of magnitude + between organisms, so a pooled distribution would not help you choose a + threshold for either.
    • + +
  • Some statistics are only defined when enough alleles were sampled at a + locus - common and singleton counts, highest minor allele frequency, and + the diversity ratios. Where an organism's sampling does not reach that + floor the value is left empty rather than reported as zero, so those genes + are simply absent from that statistic's distribution. They are excluded + from your result only if you filter on that statistic.
    • + +
  • piN/piS is normalized by Nei-Gojobori site counts derived from the + genetic code, so it does not carry the codon-bias distortion that the + older non-synonymous / synonymous count ratio does. Below 1 suggests + purifying selection; above 1 suggests diversifying selection.
    • + + ]]> +     + + + + + Display the histogram of the values of this attribute + int + + + + + Display the histogram of the values of this attribute + float + + + + + Display the histogram of the values of this attribute + int + + + + + Display the histogram of the values of this attribute + int + + + + + + + + Display the histogram of the values of this attribute + float + + + + + + +     + diff --git a/Model/lib/wdk/model/questions/params/geneParams.xml b/Model/lib/wdk/model/questions/params/geneParams.xml index dd59a46a7f..be76b3a972 100644 --- a/Model/lib/wdk/model/questions/params/geneParams.xml +++ b/Model/lib/wdk/model/questions/params/geneParams.xml @@ -2391,6 +2391,61 @@ products of your selected type (or types).

    + + + + + +
    + Statistics that depend on allele frequency - common and singleton counts, highest + minor allele frequency, and the diversity ratios - are only defined where enough + alleles were sampled at a locus. Genes below that floor carry no value and are + simply absent from that statistic's distribution rather than being counted as zero. + ]]> +     + + Filter genes by their summarized variant statistics. Choose an organism first. + +     + +
    + + + + + + + + + + + + + + + + 5%)', + 'Loci with a minor allele frequency above 5%. Counted only where at least 20 alleles were sampled at the locus; genes with no qualifying locus carry no value and are absent from this distribution.', + 'number', 'variants', 0, 1, 21), + ('n_singleton', 'variant_frequency', 'Singleton variants', + 'Loci seen in exactly one allele copy. Requires at least 100 sampled alleles at the locus.', + 'number', 'variants', 0, 1, 22), + ('max_minor_allele_frequency', 'variant_frequency', 'Highest minor allele frequency', + 'The largest minor allele frequency at any locus in the gene. 0.5 is its ceiling by definition.', + 'number', NULL, 4, 1, 23), + + ('variant_selection', NULL, 'Diversity and selection', NULL, + NULL, NULL, NULL, 0, 30), + ('pi_n_pi_s_snpeff', 'variant_selection', 'piN/piS', + 'Nonsynonymous over synonymous nucleotide diversity for the representative transcript, normalized by Nei-Gojobori site counts derived from the genetic code. Below 1 suggests purifying selection, above 1 diversifying. Unlike a raw nonsynonymous/synonymous count ratio this does not carry the codon-bias distortion. Defined only where enough alleles were sampled.', + 'number', NULL, 3, 1, 31), + ('pi_per_site_cds', 'variant_selection', 'Nucleotide diversity per CDS site', + 'Mean pairwise nucleotide diversity over the representative transcript''s coding sites.', + 'number', NULL, 6, 1, 32), + ('nonsyn_syn_ratio_snpeff', 'variant_selection', 'Nonsynonymous / synonymous count ratio', + 'The raw count ratio. Kept for continuity with the older SNP searches; piN/piS above is the normalized statistic and is the better measure.', + 'number', NULL, 2, 1, 33), + + ('variant_sampling', NULL, 'Sampling depth', NULL, + NULL, NULL, NULL, 0, 40), + ('max_called_strain_count', 'variant_sampling', 'Strains sampled', + 'The largest number of strains with a call at any locus in the gene. Use this to judge how much weight the frequency statistics can carry.', + 'number', 'strains', 0, 1, 41), + ('avg_call_rate', 'variant_sampling', 'Average call rate', + 'Mean fraction of sampled strains with a call, across the gene''s loci.', + 'number', NULL, 3, 1, 42) + ) AS t(ontology_term_name, parent_ontology_term_name, display_name, + description, type, units, precision, is_range, display_order) + ORDER BY display_order + ]]> + + + + + + + + + + + + + + + + + diff --git a/Model/lib/wdk/model/questions/params/organismParams.xml b/Model/lib/wdk/model/questions/params/organismParams.xml index d9770d3fcb..9909701092 100644 --- a/Model/lib/wdk/model/questions/params/organismParams.xml +++ b/Model/lib/wdk/model/questions/params/organismParams.xml @@ -639,6 +639,46 @@ + + + + + + + + + + + + + + + diff --git a/Model/lib/wdk/model/questions/queries/geneQueries.xml b/Model/lib/wdk/model/questions/queries/geneQueries.xml index 73b4e9fe09..9bb94690f3 100644 --- a/Model/lib/wdk/model/questions/queries/geneQueries.xml +++ b/Model/lib/wdk/model/questions/queries/geneQueries.xml @@ -2144,6 +2144,80 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/Model/lib/wdk/ontology/individuals.txt b/Model/lib/wdk/ontology/individuals.txt index 6e5ffe6181..b9159293f7 100644 --- a/Model/lib/wdk/ontology/individuals.txt +++ b/Model/lib/wdk/ontology/individuals.txt @@ -97,6 +97,7 @@ TranscriptRecordClasses.TranscriptRecordClass.GeneQuestions.GenesByReactionCompo TranscriptRecordClasses.TranscriptRecordClass.GeneQuestions.GenesByMolecularWeight http://edamontology.org/topic_0123 Protein properties TranscriptRecordClasses.TranscriptRecordClass search GeneQuestions.GenesByMolecularWeight menu webservice TranscriptRecordClasses.TranscriptRecordClass.GeneQuestions.GenesByMotifSearch http://edamontology.org/topic_0080 Sequence Analysis TranscriptRecordClasses.TranscriptRecordClass search GeneQuestions.GenesByMotifSearch menu webservice TranscriptRecordClasses.TranscriptRecordClass.GeneQuestions.GenesByNgsSnps http://edamontology.org/topic_0199 Genetic Variation TranscriptRecordClasses.TranscriptRecordClass search GeneQuestions.GenesByNgsSnps menu webservice +TranscriptRecordClasses.TranscriptRecordClass.GeneQuestions.GenesByVariantCharacteristics http://edamontology.org/topic_0199 Genetic Variation TranscriptRecordClasses.TranscriptRecordClass search GeneQuestions.GenesByVariantCharacteristics menu webservice TranscriptRecordClasses.TranscriptRecordClass.GeneQuestions.GenesByCopyNumber http://edamontology.org/topic_0199 Genetic Variation TranscriptRecordClasses.TranscriptRecordClass search GeneQuestions.GenesByCopyNumber menu webservice TranscriptRecordClasses.TranscriptRecordClass.GeneQuestions.GenesByCopyNumberComparison http://edamontology.org/topic_0199 Genetic Variation TranscriptRecordClasses.TranscriptRecordClass search GeneQuestions.GenesByCopyNumberComparison menu webservice TranscriptRecordClasses.TranscriptRecordClass.GeneQuestions.GenesByNonnuclearLocation GenomicSequenceLocationCategory GenomicSequenceLocationCategory TranscriptRecordClasses.TranscriptRecordClass search GeneQuestions.GenesByNonnuclearLocation menu webservice From 0520b48c375b90cb9168cfa238dfeca471283f62 Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Fri, 7 Aug 2026 22:58:16 -0400 Subject: [PATCH 92/98] Use the transcript record's own variation attributes, not new dynamic ones TranscriptRecordClass already publishes this tuning table's sortable subset, and gene_pi_n_pi_s there is gvs.pi_n_pi_s_snpeff - the same column the search's filter reads. The dynamic columns were showing identical numbers under parallel names, which is a divergence waiting to happen. Drops the dynamicAttributes block and the gvs_* columns from the id query, leaving it returning only the four columns WDK needs. Co-Authored-By: Claude Opus 5 --- .../lib/wdk/model/questions/geneQuestions.xml | 58 +++---------------- .../model/questions/queries/geneQueries.xml | 18 ------ 2 files changed, 8 insertions(+), 68 deletions(-) diff --git a/Model/lib/wdk/model/questions/geneQuestions.xml b/Model/lib/wdk/model/questions/geneQuestions.xml index 6d41b943fa..f22d7a3bb8 100644 --- a/Model/lib/wdk/model/questions/geneQuestions.xml +++ b/Model/lib/wdk/model/questions/geneQuestions.xml @@ -1582,9 +1582,15 @@ Find genes which are differentially expressed in an RNA-Seq experiment based on queryRef="GeneId.GenesByVariantCharacteristics" recordClassRef="TranscriptRecordClasses.TranscriptRecordClass"> + + summary="gene_product,gene_total_variants,gene_variants_per_kb,gene_variants_lof,gene_variants_missense_common,gene_pi_n_pi_s,gene_variation_strains_sampled" + sorting="gene_variants_per_kb desc,gene_total_variants desc"/> - - - - Display the histogram of the values of this attribute - int - - - - - Display the histogram of the values of this attribute - float - - - - - Display the histogram of the values of this attribute - int - - - - - Display the histogram of the values of this attribute - int - - - - - - - - Display the histogram of the values of this attribute - float - - - - - - diff --git a/Model/lib/wdk/model/questions/queries/geneQueries.xml b/Model/lib/wdk/model/questions/queries/geneQueries.xml index 9bb94690f3..eba919a68f 100644 --- a/Model/lib/wdk/model/questions/queries/geneQueries.xml +++ b/Model/lib/wdk/model/questions/queries/geneQueries.xml @@ -2178,30 +2178,12 @@ - - - - - - - - - Date: Sat, 8 Aug 2026 01:02:30 -0400 Subject: [PATCH 93/98] Declare span_snp_density on the GenesByNgsSnps process query Not just the wsColumn: both postCacheUpdateSql blocks enumerate the result columns explicitly in their INSERT and SELECT lists, so without adding it there the sibling transcripts they backfill would carry NULL span density while the matched transcripts carried a value. Required by the plugin change in ApiCommonWebService 7a61e99 - getColumns now advertises span_snp_density, and PluginExecutor.validateColumns rejects a plugin advertising a column the wsColumn list lacks. Co-Authored-By: Claude Opus 5 --- Model/lib/wdk/model/questions/queries/geneQueries.xml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/Model/lib/wdk/model/questions/queries/geneQueries.xml b/Model/lib/wdk/model/questions/queries/geneQueries.xml index eba919a68f..449df58ab5 100644 --- a/Model/lib/wdk/model/questions/queries/geneQueries.xml +++ b/Model/lib/wdk/model/questions/queries/geneQueries.xml @@ -2835,7 +2835,7 @@ -- query-specific version INSERT INTO ##WDK_CACHE_TABLE## ( wdk_instance_id, wdk_row_id, wdk_weight, source_id, gene_source_id, project_id, matched_result, cds_snp_density - , ngs_dn_ds_ratio, ngs_num_synonymous, ngs_num_non_synonymous, num_noncoding, num_nonsense, ngs_total_snps) + , span_snp_density, ngs_dn_ds_ratio, ngs_num_synonymous, ngs_num_non_synonymous, num_noncoding, num_nonsense, ngs_total_snps) WITH matches AS ( SELECT * FROM ##WDK_CACHE_TABLE## @@ -2856,7 +2856,7 @@ FROM matches ) SELECT ##WDK_CACHE_INSTANCE_ID##, row_number() OVER () + the_max, 0, mm.source_id, mm.gene_source_id, mm.project_id, 'Y' - , wct.cds_snp_density, wct.ngs_dn_ds_ratio, wct.ngs_num_synonymous, wct.ngs_num_non_synonymous, wct.num_noncoding + , wct.cds_snp_density, wct.span_snp_density, wct.ngs_dn_ds_ratio, wct.ngs_num_synonymous, wct.ngs_num_non_synonymous, wct.num_noncoding , wct.num_nonsense, wct.ngs_total_snps FROM row_offset ro, mismatches mm, ##WDK_CACHE_TABLE## wct WHERE mm.gene_source_id = wct.gene_source_id; @@ -2869,7 +2869,7 @@ -- insert missing transcripts of included genes -- query-specific version INSERT INTO ##WDK_CACHE_TABLE## - ( wdk_instance_id, wdk_row_id, wdk_weight, source_id, gene_source_id, matched_result, cds_snp_density, ngs_dn_ds_ratio + ( wdk_instance_id, wdk_row_id, wdk_weight, source_id, gene_source_id, matched_result, cds_snp_density, span_snp_density, ngs_dn_ds_ratio , ngs_num_synonymous, ngs_num_non_synonymous, num_noncoding, num_nonsense, ngs_total_snps) WITH matches AS ( SELECT * @@ -2891,7 +2891,7 @@ FROM matches ) SELECT ##WDK_CACHE_INSTANCE_ID##, row_number() OVER () + the_max, 0, mm.source_id, mm.gene_source_id, 'Y', wct.cds_snp_density - , wct.ngs_dn_ds_ratio, wct.ngs_num_synonymous, wct.ngs_num_non_synonymous, wct.num_noncoding, wct.num_nonsense + , wct.span_snp_density, wct.ngs_dn_ds_ratio, wct.ngs_num_synonymous, wct.ngs_num_non_synonymous, wct.num_noncoding, wct.num_nonsense , wct.ngs_total_snps FROM row_offset ro, mismatches mm, ##WDK_CACHE_TABLE## wct WHERE mm.gene_source_id = wct.gene_source_id @@ -2942,6 +2942,7 @@ + From 1827678b96b9a388e1bfeba1b6cb93ca0586b3b5 Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Sat, 8 Aug 2026 01:10:38 -0400 Subject: [PATCH 94/98] Label the GenesByNgsSnps result columns for what they now compute Adds SNVs per kb (gene span) alongside the now-actually-CDS density, renames the ratio to say it is site-normalized, and corrects three labels that never matched the classifier: nonsynonymous excludes stop-gained, and non-coding is really unclassified - class 0 means no product byte was available, which covers unclassifiable positions as well as genuinely non-coding ones. Co-Authored-By: Claude Opus 5 --- .../lib/wdk/model/questions/geneQuestions.xml | 44 ++++++++++++------- 1 file changed, 27 insertions(+), 17 deletions(-) diff --git a/Model/lib/wdk/model/questions/geneQuestions.xml b/Model/lib/wdk/model/questions/geneQuestions.xml index f22d7a3bb8..706a34b2db 100644 --- a/Model/lib/wdk/model/questions/geneQuestions.xml +++ b/Model/lib/wdk/model/questions/geneQuestions.xml @@ -1472,8 +1472,8 @@ Find genes which are differentially expressed in an RNA-Seq experiment based on queryRef="GeneId.GenesByNgsSnps" recordClassRef="TranscriptRecordClasses.TranscriptRecordClass"> @@ -1510,49 +1510,59 @@ Find genes which are differentially expressed in an RNA-Seq experiment based on ]]> - + Display the histogram of the values of this attribute int - - - Display the histogram of the values of this attribute - int + + + Display the histogram of the values of this attribute + float - - - Display the histogram of the values of this attribute - int + + + Display the histogram of the values of this attribute + float + + + + + Display the histogram of the values of this attribute + float - + Display the histogram of the values of this attribute int - + Display the histogram of the values of this attribute int - + Display the histogram of the values of this attribute int - + Display the histogram of the values of this attribute From 052be33c45a42f75262d73d75b0aa3ab674b6f32 Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Sat, 8 Aug 2026 01:12:30 -0400 Subject: [PATCH 95/98] Label the GenesByNgsSnps params for what they now filter Density is CDS density, the ratio is site-normalized, and the class enum terms follow the corrected column names. Internal enum values are unchanged - they are a contract with legalParams and with the filter's branches. Co-Authored-By: Claude Opus 5 --- .../wdk/model/questions/params/geneParams.xml | 36 +++++++++---------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/Model/lib/wdk/model/questions/params/geneParams.xml b/Model/lib/wdk/model/questions/params/geneParams.xml index be76b3a972..dd37155688 100644 --- a/Model/lib/wdk/model/questions/params/geneParams.xml +++ b/Model/lib/wdk/model/questions/params/geneParams.xml @@ -2326,16 +2326,16 @@ products of your selected type (or types).

    - Upper Bound on the ratio of non-synonymous to synonymous coding snps. NOTE: Leaving this parameter value empty means you don't care what the upper bound is. + Upper bound on the site-normalized dN/dS ratio. NOTE: Leaving this parameter value empty means you don't care what the upper bound is. - Lower Bound on the ratio of non-synonymous to synonymous coding snps + Nonsynonymous and synonymous counts each divided by the number of sites of that class. Below 1 suggests purifying selection. Stop-gained variants are counted as nonsense rather than nonsynonymous and so do not enter the numerator. Genes with no synonymous sites have no value and are returned only while this filter is left alone. @@ -2348,16 +2348,16 @@ products of your selected type (or types).

    - Upper Bound on the number of SNPs of the selected class. NOTE: Leaving this parameter value empty means you don't care what the upper bound is. + Upper Bound on the number of SNVs of the selected class. NOTE: Leaving this parameter value empty means you don't care what the upper bound is. - Lower Bound on the number of SNPs of the selected class + Lower Bound on the number of SNVs of the selected class @@ -2377,16 +2377,16 @@ products of your selected type (or types).

    - Find genes containing a density of coding snps greater than this parameter. Density is expressed in number of snps / KB of coding sequence + Coding variants per kilobase of coding sequence. Genes with no coding sequence have no value here and are returned only while this filter is left alone. - Find genes containing a density of coding snps less than this parameter. Density is expressed in number of snps / KB of coding sequence. NOTE: Leaving this parameter value empty means you don't care what the upper bound is. + Coding variants per kilobase of coding sequence. NOTE: Leaving this parameter value empty means you don't care what the upper bound is. @@ -2504,17 +2504,17 @@ products of your selected type (or types).

    - Choose the class of SNP you want to query on ... choose minumum and maximum numbers below + Choose the class of SNV you want to query on ... choose minumum and maximum numbers below - All SNPs + All SNVs all @@ -2522,15 +2522,15 @@ products of your selected type (or types).

    coding     - Non-Coding + Unclassified noncoding - Non-Synonymous + Missense nonsynonymous - Nonsense + Stop-gained nonsense From 06cca8039f0f3baae74661188a2712f2fcb58526 Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Sat, 8 Aug 2026 01:14:01 -0400 Subject: [PATCH 96/98] Point the GenesByNgsSnps description at the normalized ratio The PlasmoDB description promised normalized Dn/Ds in a future release; it is here. Also states in both descriptions that these statistics are sample-set scoped and so not comparable to the precomputed ones. Co-Authored-By: Claude Opus 5 --- Model/lib/wdk/model/questions/geneQuestions.xml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Model/lib/wdk/model/questions/geneQuestions.xml b/Model/lib/wdk/model/questions/geneQuestions.xml index 706a34b2db..158aaa71f9 100644 --- a/Model/lib/wdk/model/questions/geneQuestions.xml +++ b/Model/lib/wdk/model/questions/geneQuestions.xml @@ -1487,11 +1487,13 @@ Find genes which are differentially expressed in an RNA-Seq experiment based on +
  • These statistics are computed over the samples you select, so they change with your sample set and are not comparable to the precomputed values on the gene record page or in the "SNV Characteristics" search.
  • In this search, SNPs are defined as sequence differences between the selected strains. If you want to include sequence differences between the selected genomes and the reference genome, then also select the reference strain.
  • Leave the default settings to obtain genes with any type of SNP. On the Results page, you can sort by many of these parameters (e.g., number of SNPs, non-synonymous/synonymous ratio, etc).
  • To obtain genes containing SNPs of a specific class (e.g., nonsense), then select "Nonsense" in the "SNP Class" and change "Number of SNPs of above class >=" to "1" (or higher).
  • As an example, running this search to compare 3D7 and Dd2 using default parameters returns 2577 genes. Among the top 20 on the list are CLAG, AMA1, chloroquine resistance marker and multiple PfEMP1 and SURFIN genes.
    • -
  • Due to the extreme codon bias in the P. falciparum genome, the ratio of non-synonymous to synonymous SNPs within each gene is much higher than expected. This should be considered when creating queries. We are intending to calculate more reliable normalized Dn/Ds or Ka/Ks ratios in subsequent releases of PlasmoDB.
    • +
  • The dN/dS ratio reported here IS normalized by synonymous and nonsynonymous site counts, derived from the genetic code over the gene's representative transcript, so it does not carry the codon-bias inflation that a raw count ratio does. In P. falciparum the pooled synonymous-site fraction is 17.49% rather than the textbook ~25%, which is a 1.43x correction on every gene.
    • +
  • It remains a count ratio: a variant seen in one sample weights the same as one at 50% frequency. For a frequency-weighted piN/piS across every sample loaded, use the "SNV Characteristics" search.
    • @@ -1503,6 +1505,7 @@ Find genes which are differentially expressed in an RNA-Seq experiment based on +
  • These statistics are computed over the samples you select, so they change with your sample set and are not comparable to the precomputed values on the gene record page or in the "SNV Characteristics" search.
  • In this search, SNPs are defined as sequence differences between the selected strains. If you want to include sequence differences between the selected genomes and the reference genome, then also select the reference strain.
  • Leave the default settings to obtain genes with any type of SNP. On the Results page, you can sort by many of these parameters (e.g., number of SNPs, non-synonymous/synonymous ratio, etc).
  • To obtain genes containing SNPs of a specific class (e.g., nonsense), then select "Nonsense" in the "SNP Class" and change "Number of SNPs of above class >=" to "1" (or higher).
    • From 186d4847a707294ae88a1ed2b3f10e8d7fbd586c Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Sat, 8 Aug 2026 01:27:42 -0400 Subject: [PATCH 97/98] Drop the synonymous-site-fraction figures from the GenesByNgsSnps description The bullet keeps saying the ratio is site-normalized; it no longer quotes the 17.49% figure or the 1.43x factor at the user. Co-Authored-By: Claude Opus 5 --- Model/lib/wdk/model/questions/geneQuestions.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Model/lib/wdk/model/questions/geneQuestions.xml b/Model/lib/wdk/model/questions/geneQuestions.xml index 158aaa71f9..e10c83da4c 100644 --- a/Model/lib/wdk/model/questions/geneQuestions.xml +++ b/Model/lib/wdk/model/questions/geneQuestions.xml @@ -1492,7 +1492,7 @@ Find genes which are differentially expressed in an RNA-Seq experiment based on
  • Leave the default settings to obtain genes with any type of SNP. On the Results page, you can sort by many of these parameters (e.g., number of SNPs, non-synonymous/synonymous ratio, etc).
  • To obtain genes containing SNPs of a specific class (e.g., nonsense), then select "Nonsense" in the "SNP Class" and change "Number of SNPs of above class >=" to "1" (or higher).
  • As an example, running this search to compare 3D7 and Dd2 using default parameters returns 2577 genes. Among the top 20 on the list are CLAG, AMA1, chloroquine resistance marker and multiple PfEMP1 and SURFIN genes.
    • -
  • The dN/dS ratio reported here IS normalized by synonymous and nonsynonymous site counts, derived from the genetic code over the gene's representative transcript, so it does not carry the codon-bias inflation that a raw count ratio does. In P. falciparum the pooled synonymous-site fraction is 17.49% rather than the textbook ~25%, which is a 1.43x correction on every gene.
    • +
  • The dN/dS ratio reported here IS normalized by synonymous and nonsynonymous site counts, derived from the genetic code over the gene's representative transcript, so it does not carry the codon-bias inflation that a raw count ratio does.
  • It remains a count ratio: a variant seen in one sample weights the same as one at 50% frequency. For a frequency-weighted piN/piS across every sample loaded, use the "SNV Characteristics" search.
    • From 2dea52712a788e72983504da8453115148938ddc Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Sat, 8 Aug 2026 01:32:28 -0400 Subject: [PATCH 98/98] Mark the GenesByNgsSnps result columns as per-sample-group These are computed over the samples the user selected, but a results table can carry them alongside the whole-cohort columns from the gene record (Total Variants, Variants per kb, piN/piS) with nothing to tell them apart. Each now carries a (sample group) qualifier. The two density labels swap their existing parentheses for commas so no header ends up with two bracketed clauses. Co-Authored-By: Claude Opus 5 --- Model/lib/wdk/model/questions/geneQuestions.xml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Model/lib/wdk/model/questions/geneQuestions.xml b/Model/lib/wdk/model/questions/geneQuestions.xml index e10c83da4c..9e56921a9f 100644 --- a/Model/lib/wdk/model/questions/geneQuestions.xml +++ b/Model/lib/wdk/model/questions/geneQuestions.xml @@ -1513,14 +1513,14 @@ Find genes which are differentially expressed in an RNA-Seq experiment based on ]]> - + Display the histogram of the values of this attribute int - @@ -1528,7 +1528,7 @@ Find genes which are differentially expressed in an RNA-Seq experiment based on float - @@ -1536,7 +1536,7 @@ Find genes which are differentially expressed in an RNA-Seq experiment based on float - @@ -1544,28 +1544,28 @@ Find genes which are differentially expressed in an RNA-Seq experiment based on float - + Display the histogram of the values of this attribute int - + Display the histogram of the values of this attribute int - + Display the histogram of the values of this attribute int - + Display the histogram of the values of this attribute