diff --git a/Model/lib/psql/webready/orgSpecific/ChrCopyNumbers_p.psql b/Model/lib/psql/webready/orgSpecific/ChrCopyNumbers_p.psql index 339b46a1f..4878ed52e 100644 --- a/Model/lib/psql/webready/orgSpecific/ChrCopyNumbers_p.psql +++ b/Model/lib/psql/webready/orgSpecific/ChrCopyNumbers_p.psql @@ -1,22 +1,28 @@ :CREATE_AND_POPULATE + -- Mirrors the ChrCopyNumbers tuningTable in apiTuningManager.xml; any edit here + -- MUST be mirrored there. See the port design doc s7. + -- + -- Two changes: the dead PANIO_p join is gone, and the source is + -- GenomicSeqAttributes_p rather than TranscriptAttributes_p. Chromosome ploidy + -- is sequence-level, and SequencesByPloidy already reads the sequence table. SELECT DISTINCT - ta.project_id - , ta.org_abbrev - , current_timestamp as modification_date - , ta.na_sequence_id - , ta.chromosome + sa.project_id + , sa.org_abbrev + , sa.organism + , sa.taxon_id + , current_timestamp as modification_date + , sa.source_id + , sa.na_sequence_id + , sa.chromosome , ccn.chr_copy_number AS ploidy - , io.input_pan_id - , io.output_pan_id + , regexp_replace(pan.name, '_Ploidy$', '') AS eda_sample_stable_id FROM apidb.ChrCopyNumber ccn - , :SCHEMA.TranscriptAttributes_p ta - , :SCHEMA.PANIO_p io - WHERE ta.na_sequence_id = ccn.na_sequence_id - AND ta.chromosome IS NOT NULL - AND ccn.protocol_app_node_id = io.output_pan_id - and ta.org_abbrev = ':ORG_ABBREV' - and io.org_abbrev = ':ORG_ABBREV'; - - -:DECLARE_PARTITION; + , study.protocolappnode pan + , :SCHEMA.GenomicSeqAttributes_p sa + WHERE ccn.protocol_app_node_id = pan.protocol_app_node_id + AND sa.na_sequence_id = ccn.na_sequence_id + AND sa.chromosome IS NOT NULL + AND sa.org_abbrev = ':ORG_ABBREV'; + +:DECLARE_PARTITION; diff --git a/Model/lib/psql/webready/orgSpecific/ChrCopyNumbers_p_ix.psql b/Model/lib/psql/webready/orgSpecific/ChrCopyNumbers_p_ix.psql index 954b586db..b8e834a06 100644 --- a/Model/lib/psql/webready/orgSpecific/ChrCopyNumbers_p_ix.psql +++ b/Model/lib/psql/webready/orgSpecific/ChrCopyNumbers_p_ix.psql @@ -1,9 +1,6 @@ - CREATE index ChrCN_ix - ON :SCHEMA.ChrCopyNumbers_p (org_abbrev, input_pan_id, na_sequence_id) + -- Was two indexes, on (org_abbrev, input_pan_id, na_sequence_id) and + -- (org_abbrev, output_pan_id). Both pan columns are now eda_sample_stable_id, + -- which would make the second a redundant prefix of the first. + CREATE index ChrCN_ix + ON :SCHEMA.ChrCopyNumbers_p (org_abbrev, eda_sample_stable_id, na_sequence_id) ; - - - CREATE index ChrCN_output - ON :SCHEMA.ChrCopyNumbers_p (org_abbrev, output_pan_id) - ; - diff --git a/Model/lib/psql/webready/orgSpecific/GeneCopyNumbers_p.psql b/Model/lib/psql/webready/orgSpecific/GeneCopyNumbers_p.psql index ed2990f89..80bbbebe8 100644 --- a/Model/lib/psql/webready/orgSpecific/GeneCopyNumbers_p.psql +++ b/Model/lib/psql/webready/orgSpecific/GeneCopyNumbers_p.psql @@ -1,30 +1,59 @@ :CREATE_AND_POPULATE + -- Mirrors the GeneCopyNumbers tuningTable in apiTuningManager.xml, which + -- carries the searches until this file's next workflow run. Any edit here + -- MUST be mirrored there. See + -- docs/superpowers/specs/2026-08-06-genetic-variation-searches-port-design.md s7. + -- + -- The PANIO_p join was removed: study.Input has no rows, so PANIO_p is empty + -- and this table came out empty. Organism identity now comes from + -- TranscriptAttributes_p and sample identity from the protocolappnode name. + -- + -- ref_cn is computed from the annotation (same ortholog group, same chromosome) + -- and deliberately IGNORES apidb.genecopynumber.ref_copy_number, which the + -- loader computes per sample and which therefore undercounts for partially + -- covered samples. That made ref_cn vary per gene, and hit_medians GROUPs BY it, + -- so WDK rejected the answer with a row-count mismatch. + WITH grp AS ( + SELECT DISTINCT ta.gene_source_id, ta.chromosome, oga.group_id + FROM :SCHEMA.TranscriptAttributes_p ta + JOIN apidb.orthologgroupaasequence oga ON oga.aa_sequence_id = ta.aa_sequence_id + WHERE ta.org_abbrev = ':ORG_ABBREV' + ), refcn AS ( + SELECT a.gene_source_id, count(DISTINCT b.gene_source_id) AS ref_cn + FROM grp a + JOIN grp b ON b.group_id = a.group_id AND b.chromosome = a.chromosome + GROUP BY 1 + ) SELECT DISTINCT ':PROJECT_ID' as project_id , ':ORG_ABBREV' as org_abbrev + , ta.organism + , ta.taxon_id , current_timestamp as modification_date , ta.source_id , ta.gene_source_id - , REGEXP_REPLACE(pan.name, '_[A-Za-z0-9]+ (.+)$', '') AS strain + , regexp_replace(pan.name, '_GeneCNV$', '') AS eda_sample_stable_id , gcn.haploid_number AS raw_estimate - , gcn.ref_copy_number AS ref_cn + -- 1 = the gene itself, for a gene in no ortholog group (67 such in pfal). + -- CAUTION: ref_cn=1 is also the NORMAL result - 89% of pfal genes have no + -- same-chromosome paralog - so if this table is built before the orthomcl + -- load has run, it is silently indistinguishable from a correct one for most + -- rows. The tuningManager copy guards this with an externalDependency on + -- apidb.OrthologGroupAaSequence; here it is a workflow ORDERING requirement. + , COALESCE(r.ref_cn, 1) AS ref_cn , CASE WHEN (gcn.haploid_number < 0.01) THEN 0 WHEN (0.01 < gcn.haploid_number AND gcn.haploid_number < 1.85) THEN 1 ELSE round(gcn.haploid_number) END AS haploid_number , ta.chromosome , ta.na_sequence_id - , io.input_pan_id - , io.output_pan_id FROM apidb.genecopynumber gcn - , study.protocolappnode pan - , :SCHEMA.TranscriptAttributes_p ta - , :SCHEMA.PANIO_p io - WHERE gcn.protocol_app_node_id = pan.protocol_app_node_id - AND gcn.na_feature_id = ta.gene_na_feature_id - AND gcn.protocol_app_node_id = io.output_pan_id - AND (ta.gene_type = 'protein coding' or ta.gene_type = 'protein coding gene') - AND ta.org_abbrev = ':ORG_ABBREV' - AND io.org_abbrev = ':ORG_ABBREV'; - - -:DECLARE_PARTITION; + JOIN study.protocolappnode pan + ON pan.protocol_app_node_id = gcn.protocol_app_node_id + JOIN :SCHEMA.TranscriptAttributes_p ta + ON ta.gene_na_feature_id = gcn.na_feature_id + LEFT JOIN refcn r + ON r.gene_source_id = ta.gene_source_id + WHERE ta.gene_type IN ('protein coding', 'protein coding gene') + AND ta.org_abbrev = ':ORG_ABBREV'; + +:DECLARE_PARTITION; diff --git a/Model/lib/psql/webready/orgSpecific/GeneCopyNumbers_p_ix.psql b/Model/lib/psql/webready/orgSpecific/GeneCopyNumbers_p_ix.psql index d66c6e8ff..9a261f7ea 100644 --- a/Model/lib/psql/webready/orgSpecific/GeneCopyNumbers_p_ix.psql +++ b/Model/lib/psql/webready/orgSpecific/GeneCopyNumbers_p_ix.psql @@ -1,4 +1,3 @@ - CREATE INDEX GeneCN_ix - ON :SCHEMA.GeneCopyNumbers_p (org_abbrev, input_pan_id, na_sequence_id) + CREATE INDEX GeneCN_ix + ON :SCHEMA.GeneCopyNumbers_p (org_abbrev, eda_sample_stable_id, na_sequence_id) ; - diff --git a/Model/lib/psql/webready/orgSpecific/GeneVariationSummary_p.psql b/Model/lib/psql/webready/orgSpecific/GeneVariationSummary_p.psql new file mode 100644 index 000000000..bb6e05883 --- /dev/null +++ b/Model/lib/psql/webready/orgSpecific/GeneVariationSummary_p.psql @@ -0,0 +1,384 @@ +:CREATE_AND_POPULATE + -- Mirrors the GeneVariationSummary tuningTable in apiTuningManager.xml, which + -- carries the gene-record attributes until this file's next workflow run. Any + -- edit here MUST be mirrored there. Full design, validation evidence, and the + -- biologist-facing help text: + -- docs/superpowers/specs/2026-08-07-gene-variation-summary-design.md + -- + -- Replaces six attributes retired from geneRecord.xml (total_hts_snps, + -- hts_nonsynonymous_snps, hts_synonymous_snps, hts_noncoding_snps, + -- hts_stop_codon_snps, hts_nonsyn_syn_ratio), already commented out on master + -- along with the TranscriptAttributes_p psql that fed them. The old + -- "synonymous" was a RESIDUAL (total - nonsyn - stop - noncoding), so anything + -- the old pipeline failed to classify inflated it; the retired values are not + -- a target to reproduce. + -- + -- THIS FILE IS ORG-SPECIFIC. Everything below is scoped to :ORG_ABBREV, so the + -- effective ploidy, allele counts, and site fractions are all per-organism - + -- which is correct, because they genuinely differ (pfal 1.01, tbru 2.27, + -- afum 2.01). + -- + -- THREE THINGS THAT LOOK LIKE STYLE BUT ARE LOAD-BEARING: + -- + -- 1. TWO GRAINS. Display counts are per GENE (unioned across transcripts, + -- most-severe-wins). The pi statistics are confined to the REPRESENTATIVE + -- longest-CDS transcript, stored as rep_transcript_source_id. piN/piS is + -- defined for one CDS; counting variants across all transcripts while + -- normalizing by one transcript's site counts would let a variant in a + -- transcript-specific exon enter the numerator while that exon's sites + -- never enter the denominator. + -- + -- 2. SUPPRESS, NEVER DEGRADE. Frequency-derived statistics accumulate only + -- over loci clearing an allele floor (pi >= 4, common >= 20, rare >= 100) + -- and are NULL below it, never 0. Sample size is a PER-LOCUS property: + -- min(called_strain_count) is 1 in every loaded organism and 35% of pfal + -- loci have <100 alleles. At the 2-allele floor (one sample plus a + -- reference) MAF can only be 0.5, so pi degenerates to a rescaled variant + -- density. Rendering 0.00 there would reproduce the exact defect of the + -- retired hts_nonsyn_syn_ratio, which reported 0 for genes with no + -- synonymous sites and so displayed maximal signal as minimal. The + -- n_loci_* columns publish how many loci contributed. + -- + -- 3. PLOIDY IS DERIVED, NEVER HARDCODED (total_ploidy_count / + -- called_strain_count). afum measures 2.01 despite being a HAPLOID fungus + -- called as diploid; a hardcoded lookup would encode that calling bug as + -- truth, a measured value self-corrects when it is fixed. + -- + -- MAF is a true ALLELE frequency (verified to within 2e-5 against pfal using + -- total_ploidy_count as denominator), so 2p(1-p) holds at any ploidy. But + -- *_minor_allele_strain_count is a STRAIN count - different units - so + -- "singleton" is defined on allele copies, round(maf * total_ploidy_count) = 1. + -- + -- Source is apidb.VariationEffect ALONE: at gene x locus grain + -- VariationTranscriptProduct contributes 0 pairs it lacks, so the defensive + -- UNION in VariationAttributes is dead code at this grain. + -- + -- Site counts are Nei-Gojobori (1986), derived inline from the genetic code + -- rather than hardcoded, so the code table is the single source of truth. The + -- pooled synonymous-site fraction in pfal is 17.49%, NOT the textbook ~25% - + -- an AT-bias effect worth 1.43x on every gene. Without this normalization the + -- median piN/piS is 2.0 (implying genome-wide positive selection); with it, + -- 0.512 (the expected purifying-selection signature). + -- + -- COST NOTE: the cds_codons CTE expands every CDS into one row per codon. This + -- is the expensive step. Unavoidable for per-gene codon composition. + WITH code(codon, aa) AS (VALUES + ('TTT','F'),('TTC','F'),('TTA','L'),('TTG','L'), + ('CTT','L'),('CTC','L'),('CTA','L'),('CTG','L'), + ('ATT','I'),('ATC','I'),('ATA','I'),('ATG','M'), + ('GTT','V'),('GTC','V'),('GTA','V'),('GTG','V'), + ('TCT','S'),('TCC','S'),('TCA','S'),('TCG','S'), + ('CCT','P'),('CCC','P'),('CCA','P'),('CCG','P'), + ('ACT','T'),('ACC','T'),('ACA','T'),('ACG','T'), + ('GCT','A'),('GCC','A'),('GCA','A'),('GCG','A'), + ('TAT','Y'),('TAC','Y'),('TAA','*'),('TAG','*'), + ('CAT','H'),('CAC','H'),('CAA','Q'),('CAG','Q'), + ('AAT','N'),('AAC','N'),('AAA','K'),('AAG','K'), + ('GAT','D'),('GAC','D'),('GAA','E'),('GAG','E'), + ('TGT','C'),('TGC','C'),('TGA','*'),('TGG','W'), + ('CGT','R'),('CGC','R'),('CGA','R'),('CGG','R'), + ('AGT','S'),('AGC','S'),('AGA','R'),('AGG','R'), + ('GGT','G'),('GGC','G'),('GGA','G'),('GGG','G') + ), + -- Nei-Gojobori: per codon position, the fraction of the 3 possible single-base + -- changes that are synonymous. Stop codons excluded; a change creating a stop + -- is nonsynonymous (falls out, since the amino acid differs). Validated + -- against known degeneracy: ATG and TGG give 0 synonymous sites, four-fold + -- codons exactly 1.0, two-fold 1/3, TTA 0.667 (via TTA<->CTA), CGA 1.333. + nbr AS ( + SELECT c.codon, c.aa, p.pos, b.base, + overlay(c.codon placing b.base from p.pos for 1) AS mutated + FROM code c + CROSS JOIN generate_series(1,3) AS p(pos) + CROSS JOIN (VALUES ('A'),('C'),('G'),('T')) AS b(base) + WHERE c.aa <> '*' AND substr(c.codon, p.pos, 1) <> b.base + ), + posf AS ( + SELECT n.codon, n.pos, count(*) FILTER (WHERE m.aa = n.aa)::numeric / 3 AS f + FROM nbr n JOIN code m ON m.codon = n.mutated + GROUP BY 1,2 + ), + codon_sites AS ( + SELECT codon, sum(f) AS syn_sites, 3 - sum(f) AS nonsyn_sites + FROM posf GROUP BY codon + ), + rep AS ( + SELECT DISTINCT ON (gene_source_id) + gene_source_id, organism, taxon_id, chromosome, gene_na_feature_id, + transcript_source_id, na_feature_id AS rep_na_feature_id, + (gene_end_max - gene_start_min + 1) AS gene_length, cds_length + FROM :SCHEMA.TranscriptAttributes_p + WHERE org_abbrev = ':ORG_ABBREV' + ORDER BY gene_source_id, cds_length DESC NULLS LAST, transcript_source_id + ), + -- LATERAL, not a flat expansion: postgres streams each CDS's codons through + -- the aggregate instead of materializing one row per codon. Measured over all + -- 63,082 genes: 31s vs 52s flat, identical results, much smaller peak + -- footprint. A PL/pgSQL loop would do the same but forces procedural code into + -- both this file and the tuningManager mirror; declarative keeps them + -- comparable. Do not "optimize" into a flat expansion - that was 40% slower. + -- + -- The length filter guards against out-of-frame CDS rows, which would yield + -- garbage codons (477 of 63,765 genome-wide; none in the loaded organisms). + gene_sites AS ( + SELECT r.gene_source_id, s.syn_sites, s.nonsyn_sites + FROM rep r + JOIN :SCHEMA.CodingSequence_p cs ON cs.source_id = r.transcript_source_id + CROSS JOIN LATERAL ( + SELECT sum(k.syn_sites) AS syn_sites, + sum(k.nonsyn_sites) AS nonsyn_sites + FROM regexp_matches(upper(cs.sequence), '.{3}', 'g') AS m(arr) + JOIN codon_sites k ON k.codon = m.arr[1] + ) s + WHERE cs.org_abbrev = ':ORG_ABBREV' + AND length(cs.sequence) % 3 = 0 AND length(cs.sequence) >= 6 + ), + sev AS ( + SELECT e.source, t.gene_source_id, e.na_feature_id, + e.sequence_source_id, e.location, + CASE e.effect + WHEN 'frameshift_variant' THEN 1 + WHEN 'stop_gained' THEN 2 + WHEN 'stop_lost' THEN 2 + WHEN 'start_lost' THEN 2 + WHEN 'splice_acceptor_variant' THEN 3 + WHEN 'splice_donor_variant' THEN 3 + WHEN 'conservative_inframe_deletion' THEN 4 + WHEN 'disruptive_inframe_deletion' THEN 4 + WHEN 'conservative_inframe_insertion' THEN 4 + WHEN 'disruptive_inframe_insertion' THEN 4 + WHEN 'inframe_deletion_unnormalized' THEN 4 + WHEN 'inframe_insertion_unnormalized' THEN 4 + WHEN 'missense_variant' THEN 5 + WHEN 'splice_region_variant' THEN 6 + WHEN 'synonymous_variant' THEN 7 + WHEN 'stop_retained_variant' THEN 7 + WHEN 'start_retained_variant' THEN 7 + WHEN '5_prime_UTR_variant' THEN 8 + WHEN '3_prime_UTR_variant' THEN 8 + WHEN '5_prime_UTR_premature_start_codon_gain_variant' THEN 8 + WHEN 'non_coding_transcript_exon_variant' THEN 9 + WHEN 'non_coding_transcript_variant' THEN 9 + WHEN 'intron_variant' THEN 10 + ELSE 11 + END AS sev, + CASE e.impact WHEN 'HIGH' THEN 4 WHEN 'MODERATE' THEN 3 + WHEN 'LOW' THEN 2 WHEN 'MODIFIER' THEN 1 END AS imp + FROM apidb.VariationEffect e + JOIN :SCHEMA.TranscriptAttributes_p t ON t.na_feature_id = e.na_feature_id + WHERE t.org_abbrev = ':ORG_ABBREV' + ), + vf AS ( + SELECT sequence_source_id, location, variant_type, is_coding, call_rate, + called_strain_count, het_strain_count, indel_frame_effect, + total_ploidy_count AS n_alleles, + nullif(greatest(coalesce(snp_minor_allele_frequency,0), + coalesce(indel_minor_allele_frequency,0)),0) AS maf + FROM apidb.VariationFeature + ), + gene_locus AS ( + SELECT source, gene_source_id, sequence_source_id, location, + min(sev) AS sev, max(imp) AS imp + FROM sev GROUP BY 1,2,3,4 + ), + gl AS ( + SELECT g.*, v.variant_type, v.is_coding, v.call_rate, v.called_strain_count, + v.het_strain_count, v.indel_frame_effect, v.n_alleles, v.maf, + round(v.maf * v.n_alleles) AS minor_copies + FROM gene_locus g JOIN vf v + ON v.sequence_source_id = g.sequence_source_id AND v.location = g.location + ), + snp_agg AS ( + SELECT gene_source_id, + count(*) AS total_variants, + count(*) FILTER (WHERE is_coding=1) AS n_coding_loci, + count(*) FILTER (WHERE variant_type='SNV') AS n_snv, + count(*) FILTER (WHERE variant_type='INDEL') AS n_indel, + count(*) FILTER (WHERE variant_type='MIXED') AS n_mixed, + count(*) FILTER (WHERE sev=1) AS n_frameshift, + count(*) FILTER (WHERE sev=2) AS n_nonsense, + count(*) FILTER (WHERE sev=3) AS n_splice_disruptive, + count(*) FILTER (WHERE sev=4) AS n_inframe_indel, + count(*) FILTER (WHERE sev=5) AS n_missense, + count(*) FILTER (WHERE sev=6) AS n_splice_region, + count(*) FILTER (WHERE sev=7) AS n_synonymous, + count(*) FILTER (WHERE sev=8) AS n_utr, + count(*) FILTER (WHERE sev=9) AS n_noncoding_exon, + count(*) FILTER (WHERE sev=10) AS n_intron, + count(*) FILTER (WHERE sev=11) AS n_other, + count(*) FILTER (WHERE sev<=3) AS n_lof, + count(*) FILTER (WHERE indel_frame_effect='frameshift') AS n_indel_frameshift, + count(*) FILTER (WHERE imp=4) AS n_impact_high, + count(*) FILTER (WHERE imp=3) AS n_impact_moderate, + count(*) FILTER (WHERE imp=2) AS n_impact_low, + count(*) FILTER (WHERE imp=1) AS n_impact_modifier, + CASE max(imp) WHEN 4 THEN 'HIGH' WHEN 3 THEN 'MODERATE' + WHEN 2 THEN 'LOW' WHEN 1 THEN 'MODIFIER' END AS most_severe_impact, + max(called_strain_count) AS max_called_strain_count, + round(percentile_cont(0.5) WITHIN GROUP (ORDER BY called_strain_count)::numeric,0) + AS median_called_strain_count, + max(n_alleles) AS max_alleles, + round(avg(n_alleles/nullif(called_strain_count,0))::numeric,2) AS effective_ploidy, + round(avg(call_rate)::numeric,3) AS avg_call_rate, + round(min(call_rate)::numeric,3) AS min_call_rate, + count(*) FILTER (WHERE call_rate < 0.5) AS n_low_call_rate, + count(*) FILTER (WHERE het_strain_count > 0) AS n_het_loci, + count(*) FILTER (WHERE n_alleles >= 4) AS n_loci_pi, + count(*) FILTER (WHERE n_alleles >= 20) AS n_loci_freq20, + count(*) FILTER (WHERE n_alleles >= 100) AS n_loci_freq100, + count(*) FILTER (WHERE n_alleles >= 20 AND maf > 0.05) AS n_common_raw, + count(*) FILTER (WHERE n_alleles >= 20 AND maf > 0.20) AS n_very_common_raw, + count(*) FILTER (WHERE n_alleles >= 20 AND maf > 0.05 AND sev=5) AS n_missense_common_raw, + count(*) FILTER (WHERE n_alleles >= 20 AND maf > 0.05 AND sev=7) AS n_synonymous_common_raw, + count(*) FILTER (WHERE n_alleles >= 20 AND maf > 0.05 AND sev<=3) AS n_lof_common_raw, + max(maf) FILTER (WHERE n_alleles >= 20) AS max_maf_raw, + count(*) FILTER (WHERE n_alleles >= 100 AND maf <= 0.01) AS n_rare_raw, + count(*) FILTER (WHERE n_alleles >= 100 AND minor_copies = 1) AS n_singleton_raw + FROM gl WHERE source='snpeff' GROUP BY 1 + ), + pc_agg AS ( + SELECT gene_source_id, + count(*) AS pc_total_coding_variants, + count(*) FILTER (WHERE sev=5) AS pc_n_missense, + count(*) FILTER (WHERE sev=7) AS pc_n_synonymous, + count(*) FILTER (WHERE sev<=3) AS pc_n_lof, + count(*) FILTER (WHERE sev=11) AS pc_n_unclassified, + count(*) FILTER (WHERE n_alleles >= 20 AND maf > 0.05 AND sev=5) AS pc_n_missense_common_raw, + count(*) FILTER (WHERE n_alleles >= 20 AND maf > 0.05 AND sev<=3) AS pc_n_lof_common_raw + FROM gl WHERE source='product_call' GROUP BY 1 + ), + tx_locus AS ( + SELECT s.source, r.gene_source_id, s.sequence_source_id, s.location, + min(s.sev) AS sev + FROM sev s JOIN rep r ON r.rep_na_feature_id = s.na_feature_id + GROUP BY 1,2,3,4 + ), + -- unbiased heterozygosity: (n/(n-1)) * 2p(1-p). ~0.4% at pfal's 236 alleles, + -- ~9% at tbru's 12, 100% at the 2-allele floor - which is why the >= 4 filter + -- exists rather than trusting the estimator. + tx_pi AS ( + SELECT t.source, t.gene_source_id, + sum(CASE WHEN t.sev=5 THEN (v.n_alleles/(v.n_alleles-1))*2*v.maf*(1-v.maf) END) AS pi_sum_nonsyn, + sum(CASE WHEN t.sev=7 THEN (v.n_alleles/(v.n_alleles-1))*2*v.maf*(1-v.maf) END) AS pi_sum_syn, + sum((v.n_alleles/(v.n_alleles-1))*2*v.maf*(1-v.maf)) AS pi_sum_all, + count(*) FILTER (WHERE t.sev=5) AS tx_n_missense, + count(*) FILTER (WHERE t.sev=7) AS tx_n_synonymous, + count(*) AS tx_n_loci_pi + FROM tx_locus t JOIN vf v + ON v.sequence_source_id = t.sequence_source_id AND v.location = t.location + WHERE v.n_alleles >= 4 AND v.maf IS NOT NULL + GROUP BY 1,2 + ) + SELECT ':PROJECT_ID' as project_id + , ':ORG_ABBREV' as org_abbrev + , current_timestamp as modification_date + , r.organism + , r.taxon_id + , r.gene_source_id + , r.gene_na_feature_id + , r.chromosome + , r.gene_length + , r.cds_length + , r.transcript_source_id AS rep_transcript_source_id + , gs.syn_sites + , gs.nonsyn_sites + , round((gs.syn_sites/nullif(gs.syn_sites+gs.nonsyn_sites,0))::numeric,4) AS syn_site_fraction + , a.max_called_strain_count + , a.median_called_strain_count + , a.max_alleles + , a.effective_ploidy + , a.avg_call_rate + , a.min_call_rate + , a.n_low_call_rate + , a.n_het_loci + , round((a.n_het_loci::numeric/nullif(a.total_variants,0)),3) AS prop_het_loci + , a.n_loci_pi + , a.n_loci_freq20 + , a.n_loci_freq100 + , a.total_variants + , a.n_coding_loci + , a.n_snv + , a.n_indel + , a.n_mixed + , round((1000.0*a.total_variants/nullif(r.gene_length,0)),2) AS variants_per_kb + , a.n_frameshift + , a.n_nonsense + , a.n_splice_disruptive + , a.n_inframe_indel + , a.n_missense + , a.n_splice_region + , a.n_synonymous + , a.n_utr + , a.n_noncoding_exon + , a.n_intron + , a.n_other + , a.n_lof + , a.n_indel_frameshift + , a.n_impact_high + , a.n_impact_moderate + , a.n_impact_low + , a.n_impact_modifier + , a.most_severe_impact + , p.pc_total_coding_variants + , p.pc_n_missense + , p.pc_n_synonymous + , p.pc_n_lof + , p.pc_n_unclassified + -- frequency bins: NULL, not 0, when no locus cleared the floor + , CASE WHEN a.n_loci_freq20 > 0 THEN a.n_common_raw END AS n_common + , CASE WHEN a.n_loci_freq20 > 0 THEN a.n_very_common_raw END AS n_very_common + , CASE WHEN a.n_loci_freq20 > 0 THEN a.n_missense_common_raw END AS n_missense_common + , CASE WHEN a.n_loci_freq20 > 0 THEN a.n_synonymous_common_raw END AS n_synonymous_common + , CASE WHEN a.n_loci_freq20 > 0 THEN a.n_lof_common_raw END AS n_lof_common + , CASE WHEN a.n_loci_freq20 > 0 THEN p.pc_n_missense_common_raw END AS pc_n_missense_common + , CASE WHEN a.n_loci_freq20 > 0 THEN p.pc_n_lof_common_raw END AS pc_n_lof_common + , CASE WHEN a.n_loci_freq20 > 0 THEN round(a.max_maf_raw::numeric,4) END AS max_minor_allele_frequency + , CASE WHEN a.n_loci_freq100 > 0 THEN a.n_rare_raw END AS n_rare + , CASE WHEN a.n_loci_freq100 > 0 THEN a.n_singleton_raw END AS n_singleton + -- count ratios: a denominator under 5 synonymous loci is noise. NOT a + -- selection statistic - pfal's median count ratio is 2.22 while its median + -- site-normalized piN/piS is 0.512. Label as a raw count ratio on the page. + , CASE WHEN a.n_synonymous >= 5 + THEN round((a.n_missense::numeric/a.n_synonymous),2) END AS nonsyn_syn_ratio_snpeff + , CASE WHEN p.pc_n_synonymous >= 5 + THEN round((p.pc_n_missense::numeric/p.pc_n_synonymous),2) END AS nonsyn_syn_ratio_product_call + , round((se.pi_sum_nonsyn/nullif(gs.nonsyn_sites,0))::numeric,6) AS pi_nonsyn_snpeff + , round((se.pi_sum_syn /nullif(gs.syn_sites,0))::numeric,6) AS pi_syn_snpeff + -- GUARDED twin: for sorting/searching/filtering, where a gene with 2 + -- synonymous sites would otherwise dominate a descending sort. + , CASE WHEN se.tx_n_synonymous >= 5 THEN + round(((se.pi_sum_nonsyn/nullif(gs.nonsyn_sites,0)) + /nullif(se.pi_sum_syn/nullif(gs.syn_sites,0),0))::numeric,3) END AS pi_n_pi_s_snpeff + -- UNGATED twin: for DISPLAY next to tx_n_synonymous_snpeff, so the reader can + -- discount a thin denominator ("10.32, from 4 synonymous sites"). The guard + -- blanks 57.5% of pfal genes including the most-searched ones, because + -- strongly selected genes accumulate few SYNONYMOUS variants: AMA1 (4 sites), + -- PfCRT (2), Kelch13 (3) all suppress while MSP1 (28) survives, and a blank + -- reads as "no data". Cannot be replaced by the model dividing pi_nonsyn by + -- pi_syn - those are stored rounded to 6 decimals against a pi of ~1e-3, so a + -- reconstructed ratio drifts up to 0.048 (9% of genes differ), and two code + -- paths must not yield two different numbers for the same statistic. + , round(((se.pi_sum_nonsyn/nullif(gs.nonsyn_sites,0)) + /nullif(se.pi_sum_syn/nullif(gs.syn_sites,0),0))::numeric,3) AS pi_n_pi_s_snpeff_ungated + , round((pp.pi_sum_nonsyn/nullif(gs.nonsyn_sites,0))::numeric,6) AS pi_nonsyn_product_call + , round((pp.pi_sum_syn /nullif(gs.syn_sites,0))::numeric,6) AS pi_syn_product_call + , CASE WHEN pp.tx_n_synonymous >= 5 THEN + round(((pp.pi_sum_nonsyn/nullif(gs.nonsyn_sites,0)) + /nullif(pp.pi_sum_syn/nullif(gs.syn_sites,0),0))::numeric,3) END AS pi_n_pi_s_product_call + , round(((pp.pi_sum_nonsyn/nullif(gs.nonsyn_sites,0)) + /nullif(pp.pi_sum_syn/nullif(gs.syn_sites,0),0))::numeric,3) AS pi_n_pi_s_product_call_ungated + , round((se.pi_sum_all/nullif(gs.syn_sites+gs.nonsyn_sites,0))::numeric,6) AS pi_per_site_cds + , se.tx_n_missense AS tx_n_missense_snpeff + , se.tx_n_synonymous AS tx_n_synonymous_snpeff + , se.tx_n_loci_pi AS tx_n_loci_pi_snpeff + , pp.tx_n_missense AS tx_n_missense_product_call + , pp.tx_n_synonymous AS tx_n_synonymous_product_call + FROM rep r + JOIN snp_agg a ON a.gene_source_id = r.gene_source_id + LEFT JOIN pc_agg p ON p.gene_source_id = r.gene_source_id + LEFT JOIN gene_sites gs ON gs.gene_source_id = r.gene_source_id + LEFT JOIN tx_pi se ON se.gene_source_id = r.gene_source_id AND se.source = 'snpeff' + LEFT JOIN tx_pi pp ON pp.gene_source_id = r.gene_source_id AND pp.source = 'product_call'; + + +:DECLARE_PARTITION; diff --git a/Model/lib/psql/webready/orgSpecific/GeneVariationSummary_p_ix.psql b/Model/lib/psql/webready/orgSpecific/GeneVariationSummary_p_ix.psql new file mode 100644 index 000000000..0f5d1d245 --- /dev/null +++ b/Model/lib/psql/webready/orgSpecific/GeneVariationSummary_p_ix.psql @@ -0,0 +1,3 @@ + CREATE INDEX GeneVarSumm_ix + ON :SCHEMA.GeneVariationSummary_p (org_abbrev, gene_source_id) + ; diff --git a/Model/lib/wdk/apiCommonModel.xml b/Model/lib/wdk/apiCommonModel.xml index 05a812fcf..2c592ee45 100644 --- a/Model/lib/wdk/apiCommonModel.xml +++ b/Model/lib/wdk/apiCommonModel.xml @@ -476,6 +476,14 @@ Note that changing the sample subset will reset any group assignments you have a --> + + + + + + + + - + + + + + + + + + + + + + + + +
  • This search does not take a sample set. If you want to compare a + group of samples you chose - or two groups against each other - use the + "SNV Characteristics Within a Group of Samples" search instead. That one + recomputes from the raw alignments and takes read-frequency and + percent-called thresholds; this one reads precomputed values and returns + immediately.
    • + +
  • Pick an organism first. The distributions in the filter are drawn from + that organism's genes, and variant density differs by an order of magnitude + between organisms, so a pooled distribution would not help you choose a + threshold for either.
    • + +
  • Some statistics are only defined when enough alleles were sampled at a + locus - common and singleton counts, highest minor allele frequency, and + the diversity ratios. Where an organism's sampling does not reach that + floor the value is left empty rather than reported as zero, so those genes + are simply absent from that statistic's distribution. They are excluded + from your result only if you filter on that statistic.
    • + +
  • piN/piS is normalized by Nei-Gojobori site counts derived from the + genetic code, so it does not carry the codon-bias distortion that the + older non-synonymous / synonymous count ratio does. Below 1 suggests + purifying selection; above 1 suggests diversifying selection.
    • + + ]]> +     ---> @@ -4303,11 +4369,20 @@ In this study, genome-wide expression level polymorphisms (ELPs) were examined i recordClassRef="TranscriptRecordClasses.TranscriptRecordClass" includeProjects="AmoebaDB,CryptoDB,PlasmoDB,ToxoDB,TriTrypDB,FungiDB,UniDB" newBuild="24"> - ---> + summary="strains,ref_cn,median_raw_hits,median_haploid_hits,median_ploidy_hits,median_gene_dose_hits" + sorting="median_haploid_hits desc"/> Find genes based on the number of copies in resequenced strains - ---> - Find genes by comparing the gene copy number in the resequenced strain to its copy + summary="strains,ref_cn,median_raw_hits,median_haploid_hits,median_ploidy_hits,median_gene_dose_hits" + sorting="median_haploid_hits desc"/> + Find genes by comparing the gene copy number in the resequenced strain to its copy number in the reference genome.
    - Upper Bound on the ratio of non-synonymous to synonymous coding snps. NOTE: Leaving this parameter value empty means you don't care what the upper bound is. + Upper bound on the site-normalized dN/dS ratio. NOTE: Leaving this parameter value empty means you don't care what the upper bound is. - Lower Bound on the ratio of non-synonymous to synonymous coding snps + Nonsynonymous and synonymous counts each divided by the number of sites of that class. Below 1 suggests purifying selection. Stop-gained variants are counted as nonsense rather than nonsynonymous and so do not enter the numerator. Genes with no synonymous sites have no value and are returned only while this filter is left alone. @@ -2348,16 +2348,16 @@ products of your selected type (or types).

    - Upper Bound on the number of SNPs of the selected class. NOTE: Leaving this parameter value empty means you don't care what the upper bound is. + Upper Bound on the number of SNVs of the selected class. NOTE: Leaving this parameter value empty means you don't care what the upper bound is. - Lower Bound on the number of SNPs of the selected class + Lower Bound on the number of SNVs of the selected class @@ -2377,20 +2377,75 @@ products of your selected type (or types).

    - Find genes containing a density of coding snps greater than this parameter. Density is expressed in number of snps / KB of coding sequence + Coding variants per kilobase of coding sequence. Genes with no coding sequence have no value here and are returned only while this filter is left alone. - Find genes containing a density of coding snps less than this parameter. Density is expressed in number of snps / KB of coding sequence. NOTE: Leaving this parameter value empty means you don't care what the upper bound is. + Coding variants per kilobase of coding sequence. NOTE: Leaving this parameter value empty means you don't care what the upper bound is. + + + + + +
    + Statistics that depend on allele frequency - common and singleton counts, highest + minor allele frequency, and the diversity ratios - are only defined where enough + alleles were sampled at a locus. Genes below that floor carry no value and are + simply absent from that statistic's distribution rather than being counted as zero. + ]]> +     + + Filter genes by their summarized variant statistics. Choose an organism first. + +     + +
    - Choose the class of SNP you want to query on ... choose minumum and maximum numbers below + Choose the class of SNV you want to query on ... choose minumum and maximum numbers below - All SNPs + All SNVs all @@ -2467,15 +2522,15 @@ products of your selected type (or types).

    coding     - Non-Coding + Unclassified noncoding - Non-Synonymous + Missense nonsynonymous - Nonsense + Stop-gained nonsense @@ -6380,6 +6435,164 @@ products of your selected type (or types).

    + + + + + + + + + + + + + + + + 5%)', + 'Loci with a minor allele frequency above 5%. Counted only where at least 20 alleles were sampled at the locus; genes with no qualifying locus carry no value and are absent from this distribution.', + 'number', 'variants', 0, 1, 21), + ('n_singleton', 'variant_frequency', 'Singleton variants', + 'Loci seen in exactly one allele copy. Requires at least 100 sampled alleles at the locus.', + 'number', 'variants', 0, 1, 22), + ('max_minor_allele_frequency', 'variant_frequency', 'Highest minor allele frequency', + 'The largest minor allele frequency at any locus in the gene. 0.5 is its ceiling by definition.', + 'number', NULL, 4, 1, 23), + + ('variant_selection', NULL, 'Diversity and selection', NULL, + NULL, NULL, NULL, 0, 30), + ('pi_n_pi_s_snpeff', 'variant_selection', 'piN/piS', + 'Nonsynonymous over synonymous nucleotide diversity for the representative transcript, normalized by Nei-Gojobori site counts derived from the genetic code. Below 1 suggests purifying selection, above 1 diversifying. Unlike a raw nonsynonymous/synonymous count ratio this does not carry the codon-bias distortion. Defined only where enough alleles were sampled.', + 'number', NULL, 3, 1, 31), + ('pi_per_site_cds', 'variant_selection', 'Nucleotide diversity per CDS site', + 'Mean pairwise nucleotide diversity over the representative transcript''s coding sites.', + 'number', NULL, 6, 1, 32), + ('nonsyn_syn_ratio_snpeff', 'variant_selection', 'Nonsynonymous / synonymous count ratio', + 'The raw count ratio. Kept for continuity with the older SNP searches; piN/piS above is the normalized statistic and is the better measure.', + 'number', NULL, 2, 1, 33), + + ('variant_sampling', NULL, 'Sampling depth', NULL, + NULL, NULL, NULL, 0, 40), + ('max_called_strain_count', 'variant_sampling', 'Strains sampled', + 'The largest number of strains with a call at any locus in the gene. Use this to judge how much weight the frequency statistics can carry.', + 'number', 'strains', 0, 1, 41), + ('avg_call_rate', 'variant_sampling', 'Average call rate', + 'Mean fraction of sampled strains with a call, across the gene''s loci.', + 'number', NULL, 3, 1, 42) + ) AS t(ontology_term_name, parent_ontology_term_name, display_name, + description, type, units, precision, is_range, display_order) + ORDER BY display_order + ]]> + + + + + + + + + + + + + + + + + diff --git a/Model/lib/wdk/model/questions/params/organismParams.xml b/Model/lib/wdk/model/questions/params/organismParams.xml index 27627213f..28ca97341 100644 --- a/Model/lib/wdk/model/questions/params/organismParams.xml +++ b/Model/lib/wdk/model/questions/params/organismParams.xml @@ -234,25 +234,6 @@ - - - - Select the organism you wish to query against. - - - - - - - showOnlyPreferredOrganisms - - - + + - SELECT DISTINCT tn.NAME as term - , string_agg(o.abbrev, ',') as internal - FROM APIDB.DATASOURCE d - , SRES.TAXONNAME tn - , SRES.EXTERNALDATABASE ed - , SRES.EXTERNALDATABASERELEASE edr - , STUDY.nodeset s1 - , apidb.organism o - WHERE lower(d.NAME) like '%copynumbervariations_%' - AND tn.TAXON_ID = d.TAXON_ID - AND tn.taxon_id = o.taxon_id - AND tn.NAME_CLASS = 'scientific name' - AND ed.name = d.name - AND edr.VERSION = d.VERSION - AND edr.EXTERNAL_DATABASE_ID = ed.EXTERNAL_DATABASE_ID - AND s1.EXTERNAL_DATABASE_RELEASE_ID = edr.EXTERNAL_DATABASE_RELEASE_ID - GROUP BY tn.name - ORDER BY tn.NAME + + + + + + + + + + @@ -596,6 +600,85 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/Model/lib/wdk/model/questions/params/sharedParams.xml b/Model/lib/wdk/model/questions/params/sharedParams.xml index 65935765d..eecd791d2 100644 --- a/Model/lib/wdk/model/questions/params/sharedParams.xml +++ b/Model/lib/wdk/model/questions/params/sharedParams.xml @@ -1523,20 +1523,6 @@ This parameter allows you to apply the minimum number of peptides to each select - - - - - - Choose a resequenced strain or sample that has been mapped against the organism's reference genome. Genomic sequences returned by this search will be part of this strain's genome. - - - --> - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/Model/lib/wdk/model/questions/params/variantParams.xml b/Model/lib/wdk/model/questions/params/variantParams.xml new file mode 100644 index 000000000..7c6497c1b --- /dev/null +++ b/Model/lib/wdk/model/questions/params/variantParams.xml @@ -0,0 +1,663 @@ + + + + + + + + + + Input a comma delimited set of Short Variant IDs, or upload a file + + + + + + + + + + + + + Derived from the selected organism. Not user-visible. + + + + + + + + + Select a set of samples whose genomic sequences will be compared. Use the + sample characteristics to narrow the group, or accept all samples for the + organism you chose. + + + + + + + + + + Choose the resequenced strains or samples to examine. Use the sample characteristics + to narrow the group, or accept all samples for the organism you chose. + + + + + + + + + + + + + dflt + @WEBSERVICEMIRROR@/PROJECT_GOES_HERE/build-%%buildNumber%% + + + + + + + + This parameter applies to the sequencing reads of individual samples and + defines a stringency for data supporting a variant call between a sample and + the reference genome (Organism). Each nucleotide position of each sample is + compared to the reference genome and a call is made if the portion of the + sample's aligned reads that support the variant is above the Read Frequency + Threshold (RFT). Find high quality haploid variants with 80% RFT or + heterozygous diploid/aneuploid variants with 40%. See the Description below + for more. + + + + 80% + 80 + + + 60% + 60 + + + 40% + 40 + + + 20% + 20 + + + + + + + This parameter applies to your group of samples. A variant can occur in any + number of samples in your group and the least frequent call across all + samples is the Minor Allele Frequency. A variant will be returned by the + search if the frequency of the minor allele is equal to or greater than your + Minor Allele Frequency. See the Description below the Get Answer button for + more. + + + \d\d? + + + + + This parameter applies to the selected set of aligned sample sequences. At + any given nucleotide position, some samples in your group may not have data + supporting a base call because the Read Frequency Threshold was not met or + fewer than our minimum of 5 reads aligned. 'Percent samples with a base call' + defines the fraction of the selected samples that must have a base call + before a variant is returned for that nucleotide position, based on the + remaining samples that do have data. See the Description below for more + information. + + + \d\d?|100 + + + + + + + + + + + + + + + + + + + Select the first group of samples to compare. Use the sample characteristics + to narrow the group, or accept all samples for the organism you chose. + + + + + + Select the second group of samples to compare. It must differ from Set A; + comparing a group against itself returns nothing useful. + + + + + + + + + + + This parameter applies to the Set A aligned sample sequences. When a Set A + locus has a major allele frequency greater than or equal to this value, it + will be compared to the equivalent locus in Set B samples. Note that 100% is + permissible and is the most stringent setting, since the search first + identifies an allele in this set and then compares it with the allele in + Set B. See the Description below the Get Answer button for more. + + + \d\d?|100 + + + + + This parameter applies to the aligned sample sequences of Set B. When a Set B + locus has a major allele frequency greater than or equal to this value, it + will be compared to the equivalent locus in Set A samples. Note that 100% is + permissible, since the search first identifies loci from Set A and then + compares them with loci from Set B. See the Description below the Get Answer + button for more. + + + \d\d?|100 + + + + + This parameter applies to the Set B aligned sample sequences. At any given + nucleotide position, some samples in Set B may not have data supporting a + call because the Read Frequency Threshold was not met. This defines the + fraction of Set B samples that must have a base call before a locus is + returned for that position, based on the remaining samples that do have data. + See the Description below for more information. + + + \d\d?|100 + + + + + + This parameter applies to the sequencing reads of individual samples in Set B + and defines a stringency for data supporting a variant call between a sample + and the reference genome (Organism). Each nucleotide position of each sample + is compared to the reference genome and a call is made if the portion of the + sample's aligned reads that support the variant is above the Read Frequency + Threshold (RFT). Find high quality haploid variants with 80% RFT or + heterozygous diploid/aneuploid variants with 40%. See the Description below + for more. + + + + 80% + 80 + + + 60% + 60 + + + 40% + 40 + + + 20% + 20 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/Model/lib/wdk/model/questions/queries/geneQueries.xml b/Model/lib/wdk/model/questions/queries/geneQueries.xml index 1ee5ec980..89c547414 100644 --- a/Model/lib/wdk/model/questions/queries/geneQueries.xml +++ b/Model/lib/wdk/model/questions/queries/geneQueries.xml @@ -2144,6 +2144,62 @@ + + + + + + + + + + + + + + + + + + @@ -2753,7 +2809,6 @@ - + + The organism you choose will determine the samples from which you can identify SNPs. - - - SNPs are defined here as sequence differences between the selected strains. If you want to include - sequence differences between the selected strains and the reference genome, then also include the reference - strain in your search. + + + + + SNPs are defined here as sequence differences between the selected samples. If you want to include + sequence differences between the selected samples and the reference genome, then also include the reference + sample in your search. - - + + - - + + @@ -2879,6 +2942,7 @@ + @@ -2886,7 +2950,6 @@ ---> @@ -5499,8 +5562,15 @@ select distinct ta.gene_source_id - - + + + + @@ -5526,26 +5596,28 @@ select distinct ta.gene_source_id SELECT DISTINCT g.project_id , g.source_id , g.gene_source_id - , g.strain + , g.eda_sample_stable_id AS strain , g.raw_estimate , g.ref_cn , g.haploid_number , c.ploidy , g.chromosome - FROM webready.GeneCopyNumbers_p g - , webready.ChrCopyNumbers_p c - WHERE c.output_pan_id IN ($$CNV_strain$$) - AND g.input_pan_id = c.input_pan_id + FROM apidbtuning.GeneCopyNumbers g + , apidbtuning.ChrCopyNumbers c + WHERE c.eda_sample_stable_id IN ($$cnv_sample_meta$$) + AND g.eda_sample_stable_id = c.eda_sample_stable_id AND g.na_sequence_id = c.na_sequence_id - AND g.org_abbrev = $$organismSinglePickCnv$$ - AND c.org_abbrev = $$organismSinglePickCnv$$ + AND g.organism = $$organismSinglePick$$ + AND c.organism = $$organismSinglePick$$ ) , medians AS ( SELECT s.gene_source_id - , median (s.ploidy) AS median_ploidy - , median (s.raw_estimate) AS median_raw - , median (s.haploid_number) AS median_haploid - , median (s.ploidy * s.haploid_number) AS median_gene_dose + -- percentile_cont, not Oracle's median(): median() does not exist in + -- Postgres. The sibling hit_medians CTE below already uses this form. + , percentile_cont(0.5) WITHIN GROUP (ORDER BY s.ploidy) AS median_ploidy + , percentile_cont(0.5) WITHIN GROUP (ORDER BY s.raw_estimate) AS median_raw + , percentile_cont(0.5) WITHIN GROUP (ORDER BY s.haploid_number) AS median_haploid + , percentile_cont(0.5) WITHIN GROUP (ORDER BY s.ploidy * s.haploid_number) AS median_gene_dose FROM bySample s GROUP BY s.gene_source_id ) @@ -5598,8 +5670,15 @@ select distinct ta.gene_source_id - - + + + + @@ -5623,25 +5702,27 @@ select distinct ta.gene_source_id SELECT DISTINCT g.project_id , g.source_id , g.gene_source_id - , g.strain + , g.eda_sample_stable_id AS strain , g.raw_estimate , g.ref_cn , g.haploid_number , c.ploidy , g.chromosome - FROM webready.GeneCopyNumbers_p g - , webready.ChrCopyNumbers_p c - WHERE c.output_pan_id IN ($$CNV_strain$$) - AND g.input_pan_id = c.input_pan_id + FROM apidbtuning.GeneCopyNumbers g + , apidbtuning.ChrCopyNumbers c + WHERE c.eda_sample_stable_id IN ($$cnv_sample_meta$$) + AND g.eda_sample_stable_id = c.eda_sample_stable_id AND g.na_sequence_id = c.na_sequence_id - AND g.org_abbrev = $$organismSinglePickCnv$$ - AND c.org_abbrev = $$organismSinglePickCnv$$ + AND g.organism = $$organismSinglePick$$ + AND c.organism = $$organismSinglePick$$ ) , medians AS ( SELECT s.gene_source_id - , median (s.ploidy) AS median_ploidy - , median (s.raw_estimate) AS median_raw - , median (s.haploid_number) AS median_haploid - , median (s.ploidy * s.haploid_number) AS median_gene_dose + -- percentile_cont, not Oracle's median(): median() does not exist in + -- Postgres. The sibling hit_medians CTE below already uses this form. + , percentile_cont(0.5) WITHIN GROUP (ORDER BY s.ploidy) AS median_ploidy + , percentile_cont(0.5) WITHIN GROUP (ORDER BY s.raw_estimate) AS median_raw + , percentile_cont(0.5) WITHIN GROUP (ORDER BY s.haploid_number) AS median_haploid + , percentile_cont(0.5) WITHIN GROUP (ORDER BY s.ploidy * s.haploid_number) AS median_gene_dose FROM bySample s GROUP BY s.gene_source_id ) , hit_medians AS ( diff --git a/Model/lib/wdk/model/questions/queries/genomicQueries.xml b/Model/lib/wdk/model/questions/queries/genomicQueries.xml index dbb1a3031..5c69a908f 100644 --- a/Model/lib/wdk/model/questions/queries/genomicQueries.xml +++ b/Model/lib/wdk/model/questions/queries/genomicQueries.xml @@ -308,8 +308,13 @@ - - + + + + @@ -320,17 +325,13 @@ + + + + + + + + + + + + + + + + + + + + + + + + The Organism defines the species identity of the samples and the genome + against which each sample's variants were called. After choosing an Organism, + the set of samples available for forming groups is limited to samples aligned + to your chosen Organism's genome. + The organism you choose determines the genome to which the + variants have been mapped. It also restricts the set of samples you may + choose, since variants are identified by aligning that sample's reads to this + genome. + + + + + + + + + + + + + + + + + + + + + + + + The Organism defines the species identity of the samples and the genome + against which each sample's variants were called. + + + + + + + + + + + + + + + + + + + + + + + + + + + + The Organism defines the species identity of the samples and the genome + against which each sample's variants were called. + + + + + + + + + + + + + + + + + + + + + + + + + + + The Organism defines the species identity of the samples and the genome + against which each sample's variants were called. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/Model/lib/wdk/model/questions/variantQuestions.xml b/Model/lib/wdk/model/questions/variantQuestions.xml new file mode 100644 index 000000000..287fa4591 --- /dev/null +++ b/Model/lib/wdk/model/questions/variantQuestions.xml @@ -0,0 +1,391 @@ + + + + + + + + + + + + Find short variants by ID. + + + +
    + + Either enter the ID list manually, or upload a file that contains the list. + IDs can be delimited by a comma, a semi colon, or any white spaces. + ]]> +     + +     + + + + + + + + + + + + + + +
    + + Each sample's sequencing reads are aligned to the reference genome + (Organism) and variants are recorded for each sample based on the Read + Frequency Threshold. Then, scanning variant locations across the group of + samples, variants are returned by the search if the Minor Allele Frequency + and the Percent samples with a base call are met. + +

    Organism: The Organism parameter defines the species of the + samples and the genome in which the variants are determined. Choosing an + Organism focuses the Samples parameter to the samples of that organism, + changing the subset available when forming your group.

    + +

    Samples: Sample sequences are accompanied by characteristics of + the sample -- where it was collected, the host, alignment statistics. By + default the group includes all samples from the Organism you chose; you may + narrow the group using those characteristics. At least two samples are + required, since polymorphism within a group of one is undefined.

    + +

    Read frequency threshold: An allele is called for a sample at a + location if that fraction of the sample's aligned reads support it. For + example, a sample with 10 reads at a location -- 6 A and 4 C -- is called A + at a threshold of 60% or less, and not called at 80%. This matters most for + diploid or aneuploid organisms, where heterozygous positions are expected + near 50%.

    + +

    Minor allele frequency: Among the qualifying calls at a location, + the minor allele frequency is the percent that are not the major allele. A + location is returned if that is at or above the value you specify. Use 0 to + find every variant location within the group.

    + +

    Percent samples with a base call: A location is only considered if + this fraction of your selected samples have a qualifying call there. With 20 + samples and a threshold of 75%, a location with fewer than 15 called samples + is ignored.

    + ]]> +     + + + + + Display the histogram of the values of this attribute + int + + + + + Display the histogram of the values of this attribute + int + + + + + +     + + + + + + + + + + + + + +
    + + Each sample's sequencing reads are aligned to the reference genome (Organism) + and variants are recorded for each sample based on the Read Frequency + Threshold. Then, scanning variant locations within your region across the + group of samples, variants are returned if the Minor Allele Frequency and the + Percent samples with a base call are met. + +

    Defining the region: Either choose a Chromosome, or enter a Genomic + sequence ID. A sequence ID you enter takes precedence; the Chromosome menu is + used when you leave the sequence box empty. Start and End restrict the region + further, and an End of 0 means "to the end of the sequence".

    + +

    Organism: The Organism parameter defines the species of the samples + and the genome in which the variants are determined. Choosing an Organism + focuses the Samples parameter to the samples of that organism.

    + +

    Samples: By default the group includes all samples from the Organism + you chose; you may narrow it using the sample characteristics. At least two + samples are required, since polymorphism within a group of one is undefined.

    + +

    Read frequency threshold: An allele is called for a sample at a + location if that fraction of the sample's aligned reads support it. This + matters most for diploid or aneuploid organisms, where heterozygous positions + are expected near 50%.

    + +

    Minor allele frequency: Among the qualifying calls at a location, + the minor allele frequency is the percent that are not the major allele. Use 0 + to find every variant location within the group.

    + +

    Percent samples with a base call: A location is only considered if + this fraction of your selected samples have a qualifying call there.

    + ]]> +     + + + + + + Display the histogram of the values of this attribute + int + + + + + Display the histogram of the values of this attribute + int + + + + + +     + + + + + + + + + + + + + +
    + + Each sample's sequencing reads are aligned to the reference genome (Organism) + and variants are recorded for each sample based on the Read Frequency + Threshold. Then, scanning variant locations within your genes across the group + of samples, variants are returned if the Minor Allele Frequency and the Percent + samples with a base call are met. + +

    Genes: Your gene IDs are resolved to each gene's genomic span, and + variants are returned by position within those spans. A variant in an intron or + UTR of one of your genes is therefore returned, since the span covers the whole + gene rather than only its coding sequence.

    + +

    Organism: The Organism parameter defines the species of the samples + and the genome in which the variants are determined. Choose the organism your + genes belong to.

    + +

    Samples: By default the group includes all samples from the Organism + you chose; you may narrow it using the sample characteristics. At least two + samples are required, since polymorphism within a group of one is undefined.

    + +

    Read frequency threshold: An allele is called for a sample at a + location if that fraction of the sample's aligned reads support it. This + matters most for diploid or aneuploid organisms, where heterozygous positions + are expected near 50%.

    + +

    Minor allele frequency: Among the qualifying calls at a location, the + minor allele frequency is the percent that are not the major allele. Use 0 to + find every variant location within the group.

    + +

    Percent samples with a base call: A location is only considered if + this fraction of your selected samples have a qualifying call there.

    + ]]> +     + + + + + + Display the histogram of the values of this attribute + int + + + + + Display the histogram of the values of this attribute + int + + + + + +     + + + + + + + + + + + + + + +
    + + Each sample's sequencing reads are aligned to the reference genome (Organism) + and variants are recorded for each sample based on the Read Frequency + Threshold. Then, scanning locations across the samples in Set A and Set B + separately, the major allele of each set is recorded where it meets that + set's major allele frequency and percent samples with a base call. A location + is returned when the two sets' major alleles differ. + +

    Choosing the two groups: Set A and Set B must differ. Use the + sample characteristics to define each group, for example samples from two + different countries, or two different host phenotypes.

    + +

    Major allele frequency: Among the qualifying calls at a location + within one set, the major allele frequency is the percent carrying the most + common allele. Unlike the within-group searches, 100% is permissible here and + is the most stringent setting: the search identifies each set's major allele + first and then compares the two, so demanding unanimity within a set is a + sharper test rather than an impossible one. Lower the threshold to return + more locations.

    + +

    Read frequency threshold: An allele is called for a sample at a + location if that fraction of the sample's aligned reads support it. Each set + has its own threshold.

    + +

    Percent samples with a base call: A location is only considered + within a set if this fraction of that set's samples have a qualifying call + there.

    + ]]> +     + + + + + + + + + + + + + + + + + + variation_sample_meta_a + variation_sample_meta_b + + +     + +     + +     diff --git a/Model/lib/wdk/model/records/geneAttributeQueries.xml b/Model/lib/wdk/model/records/geneAttributeQueries.xml index 8e1e2dbb0..5d998d3df 100644 --- a/Model/lib/wdk/model/records/geneAttributeQueries.xml +++ b/Model/lib/wdk/model/records/geneAttributeQueries.xml @@ -146,6 +146,102 @@ WHERE ga.org_abbrev IN (%%PARTITION_KEYS%%) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/Model/lib/wdk/model/records/geneRecord.xml b/Model/lib/wdk/model/records/geneRecord.xml index 5304a70b5..01366d05f 100644 --- a/Model/lib/wdk/model/records/geneRecord.xml +++ b/Model/lib/wdk/model/records/geneRecord.xml @@ -393,6 +393,113 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/Model/lib/wdk/model/records/transcriptAttributeQueries.xml b/Model/lib/wdk/model/records/transcriptAttributeQueries.xml index 363e68dc8..b200036a5 100644 --- a/Model/lib/wdk/model/records/transcriptAttributeQueries.xml +++ b/Model/lib/wdk/model/records/transcriptAttributeQueries.xml @@ -589,6 +589,61 @@ ELSE 'N/A' end as apollo_link_out     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/Model/lib/wdk/model/records/transcriptRecord.xml b/Model/lib/wdk/model/records/transcriptRecord.xml index 946385a18..aee363de1 100644 --- a/Model/lib/wdk/model/records/transcriptRecord.xml +++ b/Model/lib/wdk/model/records/transcriptRecord.xml @@ -738,6 +738,46 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Variant_Pf3D7_01_v3_100057 + PlasmoDB + + + + Variant_11L3_v3_26886 + TriTrypDB + + + + Variant_Chr1_A_fumigatus_Af293_1000005 + FungiDB + + + + SELECT count(*) FROM ApidbTuning.VariationAttributes + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ', SUBSTR(va.organism, 1, 1), '.', + REGEXP_REPLACE(SUBSTR(va.organism, strpos(va.organism, ' ')), + '[[:space:]]+', CONCAT(chr(38), 'nbsp;')), + '') AS formatted_organism, + va.ncbi_tax_id, + va.dataset + , va.gene_ids, va.gene_count + , va.most_severe_impact_snpeff, va.most_severe_impact_product_call + , va.effect_summary_snpeff, va.effect_summary_product_call + , va.collapsed_allele, va.collapsed_minor_allele_frequency + , CASE va.most_severe_impact_snpeff + WHEN 'HIGH' THEN 4 WHEN 'MODERATE' THEN 3 + WHEN 'LOW' THEN 2 WHEN 'MODIFIER' THEN 1 END AS most_severe_impact_snpeff_rank + , CASE va.most_severe_impact_product_call + WHEN 'HIGH' THEN 4 WHEN 'MODERATE' THEN 3 + WHEN 'LOW' THEN 2 WHEN 'MODIFIER' THEN 1 END AS most_severe_impact_product_call_rank + FROM ApidbTuning.VariationAttributes va + ]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/Model/lib/wdk/model/records/variantRecords.xml b/Model/lib/wdk/model/records/variantRecords.xml new file mode 100644 index 000000000..e84038dd0 --- /dev/null +++ b/Model/lib/wdk/model/records/variantRecords.xml @@ -0,0 +1,312 @@ + + + + + + + + + Variant_Pf3D7_01_v3_100057 + PlasmoDB + + + + Variant_Pf3D7_01_v3_100057 + + + + Variant_11L3_v3_26886 + TriTrypDB + + + + Variant_Chr1_A_fumigatus_Af293_1000005 + FungiDB + + + + source_id + project_id + + + + + + + + + + + + + + 500 + + + + 1000000 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $$organism_text$$ ]]> + + + + + + +
    +
    +
    +
    Organism
    $$organism$$
    +
    Location
    $$variant_location$$
    +
    Variant Type
    $$variant_type$$
    +
    Coding
    $$is_coding$$
    +
    Reference Strain
    $$reference_strain$$
    +
    Gene(s)
    $$gene_ids$$
    +
    Most Severe Impact (SnpEff)
    $$most_severe_impact_snpeff$$
    +
    Most Severe Impact (Product Call)
    $$most_severe_impact_product_call$$
    +
    +
    +
    +
    + +
    SNP Reference
    $$snp_ref_allele$$
    +
    SNP Major
    $$snp_major_allele_and_freq$$
    +
    SNP Minor
    $$snp_minor_allele_and_freq$$
    +
    Indel Reference
    $$indel_ref_allele$$
    +
    Indel Major
    $$indel_major_allele_and_freq$$
    +
    Indel Minor
    $$indel_minor_allele_and_freq$$
    +
    Called Strain Count (including reference)
    $$distinct_strain_count$$
    +
    Called / No-Call
    $$called_strain_count$$ / $$no_call_strain_count$$
    +
    Call Rate
    $$call_rate$$
    +
    Heterozygous Strains
    $$het_strain_count$$
    +
    +
    +
    + + ]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    + + + + + + + + + + + + + + +
    + +     + +     +     diff --git a/Model/lib/wdk/model/records/variantTableQueries.xml b/Model/lib/wdk/model/records/variantTableQueries.xml new file mode 100644 index 000000000..e84e6db1f --- /dev/null +++ b/Model/lib/wdk/model/records/variantTableQueries.xml @@ -0,0 +1,82 @@ + + + + + + Variant_Pf3D7_01_v3_100057 + PlasmoDB + + + + Variant_11L3_v3_26886 + TriTrypDB + + + + Variant_Chr1_A_fumigatus_Af293_1000005 + FungiDB + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/Model/lib/wdk/ontology/individuals.txt b/Model/lib/wdk/ontology/individuals.txt index 8da8e3751..b9159293f 100644 --- a/Model/lib/wdk/ontology/individuals.txt +++ b/Model/lib/wdk/ontology/individuals.txt @@ -2,6 +2,9 @@ eupath/eupath.owl#recordClassName eupath/eupath.owl#targetType eupath/eupath.owl#name EUPATH_0000052 eupath/eupath.owl#shortDisplayName eupath/eupath.owl#description eupath/eupath.owl#geneOrTranscript EUPATH_0000274 eupath/eupath.owl#scope eupath/eupath.owl#scope eupath/eupath.owl#scope GenomicSequencePropertiesCategory GenomicSequencePropertiesCategory Genomic Sequence Properties 5 GenomicSequenceLocationCategory GenomicSequenceLocationCategory Genomic Location 6 +VariantSnpAlleleCategory http://edamontology.org/topic_2885 DNA polymorphism VariantSnpAlleleCategory SNP Alleles 1 +VariantIndelAlleleCategory http://edamontology.org/topic_2885 DNA polymorphism VariantIndelAlleleCategory Indel Alleles 2 +VariantStrainStatsCategory http://edamontology.org/topic_2885 DNA polymorphism VariantStrainStatsCategory Strain Statistics 3 TextCategory TextCategory Text 1 AlignmentsCategory http://edamontology.org/topic_0080 Sequence Analysis AlignmentsCategory BLAT and Blast Alignments CodingPotentialCategory http://edamontology.org/topic_0080 Sequence Analysis CodingPotentialCategory Coding Potential @@ -94,8 +97,9 @@ TranscriptRecordClasses.TranscriptRecordClass.GeneQuestions.GenesByReactionCompo TranscriptRecordClasses.TranscriptRecordClass.GeneQuestions.GenesByMolecularWeight http://edamontology.org/topic_0123 Protein properties TranscriptRecordClasses.TranscriptRecordClass search GeneQuestions.GenesByMolecularWeight menu webservice TranscriptRecordClasses.TranscriptRecordClass.GeneQuestions.GenesByMotifSearch http://edamontology.org/topic_0080 Sequence Analysis TranscriptRecordClasses.TranscriptRecordClass search GeneQuestions.GenesByMotifSearch menu webservice TranscriptRecordClasses.TranscriptRecordClass.GeneQuestions.GenesByNgsSnps http://edamontology.org/topic_0199 Genetic Variation TranscriptRecordClasses.TranscriptRecordClass search GeneQuestions.GenesByNgsSnps menu webservice -##TranscriptRecordClasses.TranscriptRecordClass.GeneQuestions.GenesByCopyNumber http://edamontology.org/topic_0199 Genetic Variation TranscriptRecordClasses.TranscriptRecordClass search GeneQuestions.GenesByCopyNumber menu webservice -##TranscriptRecordClasses.TranscriptRecordClass.GeneQuestions.GenesByCopyNumberComparison http://edamontology.org/topic_0199 Genetic Variation TranscriptRecordClasses.TranscriptRecordClass search GeneQuestions.GenesByCopyNumberComparison menu webservice +TranscriptRecordClasses.TranscriptRecordClass.GeneQuestions.GenesByVariantCharacteristics http://edamontology.org/topic_0199 Genetic Variation TranscriptRecordClasses.TranscriptRecordClass search GeneQuestions.GenesByVariantCharacteristics menu webservice +TranscriptRecordClasses.TranscriptRecordClass.GeneQuestions.GenesByCopyNumber http://edamontology.org/topic_0199 Genetic Variation TranscriptRecordClasses.TranscriptRecordClass search GeneQuestions.GenesByCopyNumber menu webservice +TranscriptRecordClasses.TranscriptRecordClass.GeneQuestions.GenesByCopyNumberComparison http://edamontology.org/topic_0199 Genetic Variation TranscriptRecordClasses.TranscriptRecordClass search GeneQuestions.GenesByCopyNumberComparison menu webservice TranscriptRecordClasses.TranscriptRecordClass.GeneQuestions.GenesByNonnuclearLocation GenomicSequenceLocationCategory GenomicSequenceLocationCategory TranscriptRecordClasses.TranscriptRecordClass search GeneQuestions.GenesByNonnuclearLocation menu webservice TranscriptRecordClasses.TranscriptRecordClass.GeneQuestions.GenesByOldAnnotation http://edamontology.org/topic_0219 Curation and Annotation TranscriptRecordClasses.TranscriptRecordClass search GeneQuestions.GenesByOldAnnotation TranscriptRecordClasses.TranscriptRecordClass.GeneQuestions.GenesByOrthologPattern http://edamontology.org/topic_3299 Evolutionary Biology TranscriptRecordClasses.TranscriptRecordClass search GeneQuestions.GenesByOrthologPattern menu webservice @@ -132,7 +136,7 @@ SequenceRecordClasses.SequenceRecordClass.GenomicSequenceQuestions.SequenceByWei SequenceRecordClasses.SequenceRecordClass.GenomicSequenceQuestions.SequencesBySimilarity http://edamontology.org/topic_0080 Sequence Analysis SequenceRecordClasses.SequenceRecordClass search GenomicSequenceQuestions.SequencesBySimilarity webservice SequenceRecordClasses.SequenceRecordClass.GenomicSequenceQuestions.SequencesByMultiBlast http://edamontology.org/topic_0080 Sequence Analysis SequenceRecordClasses.SequenceRecordClass search GenomicSequenceQuestions.SequencesByMultiBlast menu webservice SequenceRecordClasses.SequenceRecordClass.GenomicSequenceQuestions.SequencesByTaxon http://edamontology.org/topic_0637 Taxonomy SequenceRecordClasses.SequenceRecordClass search GenomicSequenceQuestions.SequencesByTaxon menu webservice -##SequenceRecordClasses.SequenceRecordClass.GenomicSequenceQuestions.SequencesByPloidy http://edamontology.org/topic_0219 Curation and Annotation SequenceRecordClasses.SequenceRecordClass search GenomicSequenceQuestions.SequencesByPloidy menu webservice +SequenceRecordClasses.SequenceRecordClass.GenomicSequenceQuestions.SequencesByPloidy http://edamontology.org/topic_0199 Genetic Variation SequenceRecordClasses.SequenceRecordClass search GenomicSequenceQuestions.SequencesByPloidy menu webservice TranscriptRecordClasses.TranscriptRecordClass.InternalGeneDatasetQuestions.GenesByProteinArray http://edamontology.org/topic_0804 Immunology TranscriptRecordClasses.TranscriptRecordClass search InternalGeneDatasetQuestions.GenesByProteinArray menu TranscriptRecordClasses.TranscriptRecordClass.InternalGeneDatasetQuestions.GenesByMicroarrayEvidence http://edamontology.org/topic_3308 Transcriptomics TranscriptRecordClasses.TranscriptRecordClass search InternalGeneDatasetQuestions.GenesByMicroarrayEvidence menu TranscriptRecordClasses.TranscriptRecordClass.InternalGeneDatasetQuestions.GenesByQuantitativeProteomics http://edamontology.org/topic_0121 Proteomics TranscriptRecordClasses.TranscriptRecordClass search InternalGeneDatasetQuestions.GenesByQuantitativeProteomics menu @@ -464,6 +468,52 @@ GeneRecordClasses.GeneRecordClass.hts_synonymous_snps http://edamontology.org/to GeneRecordClasses.GeneRecordClass.hts_noncoding_snps http://edamontology.org/topic_2885 DNA Polymorphism GeneRecordClasses.GeneRecordClass attribute hts_noncoding_snps gene 1 results record download GeneRecordClasses.GeneRecordClass.hts_stop_codon_snps http://edamontology.org/topic_2885 DNA Polymorphism GeneRecordClasses.GeneRecordClass attribute hts_stop_codon_snps gene 1 results record download GeneRecordClasses.GeneRecordClass.hts_nonsyn_syn_ratio http://edamontology.org/topic_2885 DNA Polymorphism GeneRecordClasses.GeneRecordClass attribute hts_nonsyn_syn_ratio gene 1 results record download +GeneVariationBasisCategory http://edamontology.org/topic_2885 DNA polymorphism GeneVariationBasisCategory Sample Basis 10 +GeneVariationCountsCategory http://edamontology.org/topic_2885 DNA polymorphism GeneVariationCountsCategory Variant Counts 11 +GeneVariationImpactCategory http://edamontology.org/topic_2885 DNA polymorphism GeneVariationImpactCategory Predicted Impact 12 +GeneVariationConsequenceCategory http://edamontology.org/topic_2885 DNA polymorphism GeneVariationConsequenceCategory Predicted Consequences 13 +GeneVariationLofCategory http://edamontology.org/topic_2885 DNA polymorphism GeneVariationLofCategory Loss of Function 14 +GeneVariationSelectionCategory http://edamontology.org/topic_2885 DNA polymorphism GeneVariationSelectionCategory Diversity and Selection 15 +GeneRecordClasses.GeneRecordClass.variation_strains_sampled GeneVariationBasisCategory Sample Basis GeneRecordClasses.GeneRecordClass attribute variation_strains_sampled gene 1 record download +GeneRecordClasses.GeneRecordClass.variation_effective_ploidy GeneVariationBasisCategory Sample Basis GeneRecordClasses.GeneRecordClass attribute variation_effective_ploidy gene 2 record download +GeneRecordClasses.GeneRecordClass.variation_call_rate GeneVariationBasisCategory Sample Basis GeneRecordClasses.GeneRecordClass attribute variation_call_rate gene 3 record download +GeneRecordClasses.GeneRecordClass.total_variants GeneVariationCountsCategory Variant Counts GeneRecordClasses.GeneRecordClass attribute total_variants gene 1 record download +GeneRecordClasses.GeneRecordClass.variants_per_kb GeneVariationCountsCategory Variant Counts GeneRecordClasses.GeneRecordClass attribute variants_per_kb gene 2 record download +GeneRecordClasses.GeneRecordClass.variant_snvs GeneVariationCountsCategory Variant Counts GeneRecordClasses.GeneRecordClass attribute variant_snvs gene 3 record download +GeneRecordClasses.GeneRecordClass.variant_indels GeneVariationCountsCategory Variant Counts GeneRecordClasses.GeneRecordClass attribute variant_indels gene 4 record download +GeneRecordClasses.GeneRecordClass.variant_mixed GeneVariationCountsCategory Variant Counts GeneRecordClasses.GeneRecordClass attribute variant_mixed gene 5 record download +GeneRecordClasses.GeneRecordClass.variants_impact_high GeneVariationImpactCategory Predicted Impact GeneRecordClasses.GeneRecordClass attribute variants_impact_high gene 1 record download +GeneRecordClasses.GeneRecordClass.variants_impact_moderate GeneVariationImpactCategory Predicted Impact GeneRecordClasses.GeneRecordClass attribute variants_impact_moderate gene 2 record download +GeneRecordClasses.GeneRecordClass.variants_impact_low GeneVariationImpactCategory Predicted Impact GeneRecordClasses.GeneRecordClass attribute variants_impact_low gene 3 record download +GeneRecordClasses.GeneRecordClass.variants_impact_modifier GeneVariationImpactCategory Predicted Impact GeneRecordClasses.GeneRecordClass attribute variants_impact_modifier gene 4 record download +GeneRecordClasses.GeneRecordClass.variants_missense GeneVariationConsequenceCategory Predicted Consequences GeneRecordClasses.GeneRecordClass attribute variants_missense gene 1 record download +GeneRecordClasses.GeneRecordClass.variants_synonymous GeneVariationConsequenceCategory Predicted Consequences GeneRecordClasses.GeneRecordClass attribute variants_synonymous gene 2 record download +GeneRecordClasses.GeneRecordClass.variants_nonsense GeneVariationConsequenceCategory Predicted Consequences GeneRecordClasses.GeneRecordClass attribute variants_nonsense gene 3 record download +GeneRecordClasses.GeneRecordClass.variants_frameshift GeneVariationConsequenceCategory Predicted Consequences GeneRecordClasses.GeneRecordClass attribute variants_frameshift gene 4 record download +GeneRecordClasses.GeneRecordClass.variants_splice_disruptive GeneVariationConsequenceCategory Predicted Consequences GeneRecordClasses.GeneRecordClass attribute variants_splice_disruptive gene 5 record download +GeneRecordClasses.GeneRecordClass.variants_inframe_indel GeneVariationConsequenceCategory Predicted Consequences GeneRecordClasses.GeneRecordClass attribute variants_inframe_indel gene 6 record download +GeneRecordClasses.GeneRecordClass.variants_utr GeneVariationConsequenceCategory Predicted Consequences GeneRecordClasses.GeneRecordClass attribute variants_utr gene 7 record download +GeneRecordClasses.GeneRecordClass.variants_intron GeneVariationConsequenceCategory Predicted Consequences GeneRecordClasses.GeneRecordClass attribute variants_intron gene 8 record download +GeneRecordClasses.GeneRecordClass.variants_lof GeneVariationLofCategory Loss of Function GeneRecordClasses.GeneRecordClass attribute variants_lof gene 1 record download +GeneRecordClasses.GeneRecordClass.variants_lof_common GeneVariationLofCategory Loss of Function GeneRecordClasses.GeneRecordClass attribute variants_lof_common gene 2 record download +GeneRecordClasses.GeneRecordClass.pi_per_site GeneVariationSelectionCategory Diversity and Selection GeneRecordClasses.GeneRecordClass attribute pi_per_site gene 1 record download +GeneRecordClasses.GeneRecordClass.pi_n_pi_s GeneVariationSelectionCategory Diversity and Selection GeneRecordClasses.GeneRecordClass attribute pi_n_pi_s gene 2 record download +GeneRecordClasses.GeneRecordClass.pi_syn_sites_used GeneVariationSelectionCategory Diversity and Selection GeneRecordClasses.GeneRecordClass attribute pi_syn_sites_used gene 3 record download +GeneRecordClasses.GeneRecordClass.variants_common GeneVariationSelectionCategory Diversity and Selection GeneRecordClasses.GeneRecordClass attribute variants_common gene 4 record download +GeneRecordClasses.GeneRecordClass.variants_missense_common GeneVariationSelectionCategory Diversity and Selection GeneRecordClasses.GeneRecordClass attribute variants_missense_common gene 5 record download +GeneRecordClasses.GeneRecordClass.variants_singleton GeneVariationSelectionCategory Diversity and Selection GeneRecordClasses.GeneRecordClass attribute variants_singleton gene 6 record download +GeneRecordClasses.GeneRecordClass.pi_n_pi_s_product_call GeneVariationSelectionCategory Diversity and Selection GeneRecordClasses.GeneRecordClass attribute pi_n_pi_s_product_call gene 7 record download +GeneRecordClasses.GeneRecordClass.nonsyn_syn_count_ratio GeneVariationSelectionCategory Diversity and Selection GeneRecordClasses.GeneRecordClass attribute nonsyn_syn_count_ratio gene 8 record download +TranscriptRecordClasses.TranscriptRecordClass.gene_variation_strains_sampled GeneVariationBasisCategory Sample Basis TranscriptRecordClasses.TranscriptRecordClass attribute gene_variation_strains_sampled gene 1 results download +TranscriptRecordClasses.TranscriptRecordClass.gene_variation_call_rate GeneVariationBasisCategory Sample Basis TranscriptRecordClasses.TranscriptRecordClass attribute gene_variation_call_rate gene 2 results download +TranscriptRecordClasses.TranscriptRecordClass.gene_total_variants GeneVariationCountsCategory Variant Counts TranscriptRecordClasses.TranscriptRecordClass attribute gene_total_variants gene 1 results download +TranscriptRecordClasses.TranscriptRecordClass.gene_variants_per_kb GeneVariationCountsCategory Variant Counts TranscriptRecordClasses.TranscriptRecordClass attribute gene_variants_per_kb gene 2 results download +TranscriptRecordClasses.TranscriptRecordClass.gene_variants_impact_high GeneVariationImpactCategory Predicted Impact TranscriptRecordClasses.TranscriptRecordClass attribute gene_variants_impact_high gene 1 results download +TranscriptRecordClasses.TranscriptRecordClass.gene_variants_lof GeneVariationLofCategory Loss of Function TranscriptRecordClasses.TranscriptRecordClass attribute gene_variants_lof gene 1 results download +TranscriptRecordClasses.TranscriptRecordClass.gene_variants_lof_common GeneVariationLofCategory Loss of Function TranscriptRecordClasses.TranscriptRecordClass attribute gene_variants_lof_common gene 2 results download +TranscriptRecordClasses.TranscriptRecordClass.gene_pi_per_site GeneVariationSelectionCategory Diversity and Selection TranscriptRecordClasses.TranscriptRecordClass attribute gene_pi_per_site gene 1 results download +TranscriptRecordClasses.TranscriptRecordClass.gene_pi_n_pi_s GeneVariationSelectionCategory Diversity and Selection TranscriptRecordClasses.TranscriptRecordClass attribute gene_pi_n_pi_s gene 2 results download +TranscriptRecordClasses.TranscriptRecordClass.gene_variants_missense_common GeneVariationSelectionCategory Diversity and Selection TranscriptRecordClasses.TranscriptRecordClass attribute gene_variants_missense_common gene 3 results download GeneRecordClasses.GeneRecordClass.uniprot_id http://edamontology.org/topic_3345 Data identity and mapping GeneRecordClasses.GeneRecordClass attribute uniprot_id gene GeneRecordClasses.GeneRecordClass.uniprot_id_internal http://edamontology.org/topic_3345 Data identity and mapping GeneRecordClasses.GeneRecordClass attribute uniprot_id_internal gene TranscriptRecordClasses.TranscriptRecordClass.uniprot_links http://edamontology.org/topic_3345 Data identity and mapping TranscriptRecordClasses.TranscriptRecordClass attribute uniprot_links transcript results @@ -1099,3 +1149,62 @@ JbrowseRecordClasses.JbrowseGeneRecordClass.location_text GenomicSequenceLocatio JbrowseRecordClasses.JbrowseGeneRecordClass.GOTerms http://edamontology.org/topic_1775 Function analysis JbrowseRecordClasses.JbrowseGeneRecordClass table GOTerms transcript record JbrowseRecordClasses.Jbrowse.GeneRecordClass.GeneTranscripts http://edamontology.org/topic_0114 Gene Structure JbrowseRecordClasses.JbrowseGeneRecordClass table GeneTranscripts gene record +VariantRecordClasses.VariantRecordClass.variant_location GenomicSequenceLocationCategory GenomicSequenceLocationCategory VariantRecordClasses.VariantRecordClass attribute variant_location results record download +VariantRecordClasses.VariantRecordClass.sequence_source_id GenomicSequenceLocationCategory GenomicSequenceLocationCategory VariantRecordClasses.VariantRecordClass attribute sequence_source_id results record download +VariantRecordClasses.VariantRecordClass.location GenomicSequenceLocationCategory GenomicSequenceLocationCategory VariantRecordClasses.VariantRecordClass attribute location results record download +VariantRecordClasses.VariantRecordClass.location_text GenomicSequenceLocationCategory GenomicSequenceLocationCategory VariantRecordClasses.VariantRecordClass attribute location_text record-internal +VariantRecordClasses.VariantRecordClass.chromosome_order_num GenomicSequenceLocationCategory GenomicSequenceLocationCategory VariantRecordClasses.VariantRecordClass attribute chromosome_order_num record-internal +VariantRecordClasses.VariantRecordClass.organism GenomicSequenceLocationCategory GenomicSequenceLocationCategory VariantRecordClasses.VariantRecordClass attribute organism results record download +VariantRecordClasses.VariantRecordClass.organism_text GenomicSequenceLocationCategory GenomicSequenceLocationCategory VariantRecordClasses.VariantRecordClass attribute organism_text record-internal +VariantRecordClasses.VariantRecordClass.formatted_organism GenomicSequenceLocationCategory GenomicSequenceLocationCategory VariantRecordClasses.VariantRecordClass attribute formatted_organism record-internal +VariantRecordClasses.VariantRecordClass.ncbi_tax_id GenomicSequenceLocationCategory GenomicSequenceLocationCategory VariantRecordClasses.VariantRecordClass attribute ncbi_tax_id record download +VariantRecordClasses.VariantRecordClass.dataset GenomicSequenceLocationCategory GenomicSequenceLocationCategory VariantRecordClasses.VariantRecordClass attribute dataset results record download +VariantRecordClasses.VariantRecordClass.variant_type http://edamontology.org/topic_2885 DNA Polymorphism VariantRecordClasses.VariantRecordClass attribute variant_type results record download +VariantRecordClasses.VariantRecordClass.is_coding http://edamontology.org/topic_2885 DNA Polymorphism VariantRecordClasses.VariantRecordClass attribute is_coding results record download +VariantRecordClasses.VariantRecordClass.reference_strain http://edamontology.org/topic_2885 DNA Polymorphism VariantRecordClasses.VariantRecordClass attribute reference_strain results record download +VariantRecordClasses.VariantRecordClass.record_overview http://edamontology.org/topic_0219 annot and curation VariantRecordClasses.VariantRecordClass attribute record_overview record-internal +VariantRecordClasses.VariantRecordClass.snp_ref_allele VariantSnpAlleleCategory SNP Alleles VariantRecordClasses.VariantRecordClass attribute snp_ref_allele 1 results record download +VariantRecordClasses.VariantRecordClass.snp_major_allele VariantSnpAlleleCategory SNP Alleles VariantRecordClasses.VariantRecordClass attribute snp_major_allele 2 results record download +VariantRecordClasses.VariantRecordClass.snp_major_allele_frequency VariantSnpAlleleCategory SNP Alleles VariantRecordClasses.VariantRecordClass attribute snp_major_allele_frequency 3 results record download +VariantRecordClasses.VariantRecordClass.snp_major_allele_strain_count VariantSnpAlleleCategory SNP Alleles VariantRecordClasses.VariantRecordClass attribute snp_major_allele_strain_count 4 results record download +VariantRecordClasses.VariantRecordClass.snp_minor_allele VariantSnpAlleleCategory SNP Alleles VariantRecordClasses.VariantRecordClass attribute snp_minor_allele 6 results record download +VariantRecordClasses.VariantRecordClass.snp_minor_allele_frequency VariantSnpAlleleCategory SNP Alleles VariantRecordClasses.VariantRecordClass attribute snp_minor_allele_frequency 7 results record download +VariantRecordClasses.VariantRecordClass.snp_minor_allele_strain_count VariantSnpAlleleCategory SNP Alleles VariantRecordClasses.VariantRecordClass attribute snp_minor_allele_strain_count 8 results record download +VariantRecordClasses.VariantRecordClass.snp_major_genomic_hgvs VariantSnpAlleleCategory SNP Alleles VariantRecordClasses.VariantRecordClass attribute snp_major_genomic_hgvs 10 results record download +VariantRecordClasses.VariantRecordClass.snp_minor_genomic_hgvs VariantSnpAlleleCategory SNP Alleles VariantRecordClasses.VariantRecordClass attribute snp_minor_genomic_hgvs 11 results record download +VariantRecordClasses.VariantRecordClass.snp_major_allele_and_freq VariantSnpAlleleCategory SNP Alleles VariantRecordClasses.VariantRecordClass attribute snp_major_allele_and_freq 5 record download +VariantRecordClasses.VariantRecordClass.snp_minor_allele_and_freq VariantSnpAlleleCategory SNP Alleles VariantRecordClasses.VariantRecordClass attribute snp_minor_allele_and_freq 9 record download +VariantRecordClasses.VariantRecordClass.indel_ref_allele VariantIndelAlleleCategory Indel Alleles VariantRecordClasses.VariantRecordClass attribute indel_ref_allele 1 results record download +VariantRecordClasses.VariantRecordClass.indel_major_allele VariantIndelAlleleCategory Indel Alleles VariantRecordClasses.VariantRecordClass attribute indel_major_allele 2 results record download +VariantRecordClasses.VariantRecordClass.indel_major_allele_frequency VariantIndelAlleleCategory Indel Alleles VariantRecordClasses.VariantRecordClass attribute indel_major_allele_frequency 3 results record download +VariantRecordClasses.VariantRecordClass.indel_major_allele_strain_count VariantIndelAlleleCategory Indel Alleles VariantRecordClasses.VariantRecordClass attribute indel_major_allele_strain_count 4 results record download +VariantRecordClasses.VariantRecordClass.indel_minor_allele VariantIndelAlleleCategory Indel Alleles VariantRecordClasses.VariantRecordClass attribute indel_minor_allele 6 results record download +VariantRecordClasses.VariantRecordClass.indel_minor_allele_frequency VariantIndelAlleleCategory Indel Alleles VariantRecordClasses.VariantRecordClass attribute indel_minor_allele_frequency 7 results record download +VariantRecordClasses.VariantRecordClass.indel_minor_allele_strain_count VariantIndelAlleleCategory Indel Alleles VariantRecordClasses.VariantRecordClass attribute indel_minor_allele_strain_count 8 results record download +VariantRecordClasses.VariantRecordClass.indel_major_genomic_hgvs VariantIndelAlleleCategory Indel Alleles VariantRecordClasses.VariantRecordClass attribute indel_major_genomic_hgvs 10 results record download +VariantRecordClasses.VariantRecordClass.indel_minor_genomic_hgvs VariantIndelAlleleCategory Indel Alleles VariantRecordClasses.VariantRecordClass attribute indel_minor_genomic_hgvs 11 results record download +VariantRecordClasses.VariantRecordClass.indel_major_allele_and_freq VariantIndelAlleleCategory Indel Alleles VariantRecordClasses.VariantRecordClass attribute indel_major_allele_and_freq 5 record download +VariantRecordClasses.VariantRecordClass.indel_minor_allele_and_freq VariantIndelAlleleCategory Indel Alleles VariantRecordClasses.VariantRecordClass attribute indel_minor_allele_and_freq 9 record download +VariantRecordClasses.VariantRecordClass.indel_frame_effect VariantIndelAlleleCategory Indel Alleles VariantRecordClasses.VariantRecordClass attribute indel_frame_effect 12 results record download +VariantRecordClasses.VariantRecordClass.distinct_strain_count VariantStrainStatsCategory Strain Statistics VariantRecordClasses.VariantRecordClass attribute distinct_strain_count 1 results record download +VariantRecordClasses.VariantRecordClass.called_strain_count VariantStrainStatsCategory Strain Statistics VariantRecordClasses.VariantRecordClass attribute called_strain_count 2 results record download +VariantRecordClasses.VariantRecordClass.no_call_strain_count VariantStrainStatsCategory Strain Statistics VariantRecordClasses.VariantRecordClass attribute no_call_strain_count 3 results record download +VariantRecordClasses.VariantRecordClass.call_rate VariantStrainStatsCategory Strain Statistics VariantRecordClasses.VariantRecordClass attribute call_rate 4 results record download +VariantRecordClasses.VariantRecordClass.total_ploidy_count VariantStrainStatsCategory Strain Statistics VariantRecordClasses.VariantRecordClass attribute total_ploidy_count 5 results record download +VariantRecordClasses.VariantRecordClass.het_strain_count VariantStrainStatsCategory Strain Statistics VariantRecordClasses.VariantRecordClass attribute het_strain_count 6 results record download +VariantRecordClasses.VariantRecordClass.ref_allele_frequency VariantStrainStatsCategory Strain Statistics VariantRecordClasses.VariantRecordClass attribute ref_allele_frequency 7 results record download +VariantRecordClasses.VariantRecordClass.gene_ids http://edamontology.org/topic_0199 Genetic Variation VariantRecordClasses.VariantRecordClass attribute gene_ids results record download +VariantRecordClasses.VariantRecordClass.most_severe_impact_snpeff http://edamontology.org/topic_0199 Genetic Variation VariantRecordClasses.VariantRecordClass attribute most_severe_impact_snpeff results record download +VariantRecordClasses.VariantRecordClass.most_severe_impact_product_call http://edamontology.org/topic_0199 Genetic Variation VariantRecordClasses.VariantRecordClass attribute most_severe_impact_product_call results record download +VariantRecordClasses.VariantRecordClass.effect_summary_snpeff http://edamontology.org/topic_0199 Genetic Variation VariantRecordClasses.VariantRecordClass attribute effect_summary_snpeff results record download +VariantRecordClasses.VariantRecordClass.effect_summary_product_call http://edamontology.org/topic_0199 Genetic Variation VariantRecordClasses.VariantRecordClass attribute effect_summary_product_call results record download +VariantRecordClasses.VariantRecordClass.collapsed_allele http://edamontology.org/topic_0199 Genetic Variation VariantRecordClasses.VariantRecordClass attribute collapsed_allele results record download +VariantRecordClasses.VariantRecordClass.collapsed_minor_allele_frequency http://edamontology.org/topic_0199 Genetic Variation VariantRecordClasses.VariantRecordClass attribute collapsed_minor_allele_frequency results record download +VariantRecordClasses.VariantRecordClass.gene_count http://edamontology.org/topic_0199 Genetic Variation VariantRecordClasses.VariantRecordClass attribute gene_count record-internal +VariantRecordClasses.VariantRecordClass.TranscriptProducts http://edamontology.org/topic_0199 Genetic Variation VariantRecordClasses.VariantRecordClass table TranscriptProducts record download +VariantRecordClasses.VariantRecordClass.PredictedEffects http://edamontology.org/topic_0199 Genetic Variation VariantRecordClasses.VariantRecordClass table PredictedEffects record download +VariantRecordClasses.VariantRecordClass.VariantQuestions.VariantBySourceId http://edamontology.org/topic_0199 Genetic Variation VariantRecordClasses.VariantRecordClass search VariantQuestions.VariantBySourceId menu webservice +VariantRecordClasses.VariantRecordClass.VariantQuestions.VariantsByIsolateGroup http://edamontology.org/topic_0199 Genetic Variation VariantRecordClasses.VariantRecordClass search VariantQuestions.VariantsByIsolateGroup menu webservice +VariantRecordClasses.VariantRecordClass.VariantQuestions.VariantsByLocation http://edamontology.org/topic_0199 Genetic Variation VariantRecordClasses.VariantRecordClass search VariantQuestions.VariantsByLocation menu webservice +VariantRecordClasses.VariantRecordClass.VariantQuestions.VariantsByGeneIds http://edamontology.org/topic_0199 Genetic Variation VariantRecordClasses.VariantRecordClass search VariantQuestions.VariantsByGeneIds menu webservice +VariantRecordClasses.VariantRecordClass.VariantQuestions.VariantsByTwoIsolateGroups http://edamontology.org/topic_0199 Genetic Variation VariantRecordClasses.VariantRecordClass search VariantQuestions.VariantsByTwoIsolateGroups menu webservice diff --git a/Model/lib/xml/tuningManager/apiTuningManager.xml b/Model/lib/xml/tuningManager/apiTuningManager.xml index 4672909e5..f5067ed27 100644 --- a/Model/lib/xml/tuningManager/apiTuningManager.xml +++ b/Model/lib/xml/tuningManager/apiTuningManager.xml @@ -511,6 +511,487 @@ + + + One row per variant locus, for the "variation" WDK record. Holds ONLY columns that + are derived, aggregated, or require a join; intrinsic per-locus facts are read + directly from apidb.VariationFeature by the WDK attribute query, since that table + is already one row per locus with a unique source_id. + + Both internalDependency elements are load-bearing: built before + TranscriptAttributes the gene aggregate is silently empty rather than an error, and + built before GenomicSeqAttributes the project_id/organism join drops every row. + + See docs/superpowers/specs/2026-07-30-variation-record-design.md in + agentic-veupath-dev. + + + + + + + + + + + ', v.snp_minor_allele) END, + CASE WHEN v.indel_minor_allele IS NOT NULL THEN concat(v.indel_ref_allele, '>', v.indel_minor_allele) END + ) AS collapsed_allele, + CASE WHEN v.snp_minor_allele_frequency IS NULL + AND v.indel_minor_allele_frequency IS NULL THEN NULL + ELSE greatest(coalesce(v.snp_minor_allele_frequency, 0), + coalesce(v.indel_minor_allele_frequency, 0)) END AS collapsed_minor_allele_frequency + FROM apidb.VariationFeature v + JOIN GenomicSeqAttributes g ON g.source_id = v.sequence_source_id + JOIN sres.ExternalDatabaseRelease r ON r.external_database_release_id = v.external_database_release_id + JOIN sres.ExternalDatabase d ON d.external_database_id = r.external_database_id + LEFT JOIN DatasetPresenter dp ON dp.name = d.name + LEFT JOIN gene_agg ga ON ga.sequence_source_id = v.sequence_source_id AND ga.location = v.location + LEFT JOIN effect_agg ea ON ea.sequence_source_id = v.sequence_source_id AND ea.location = v.location + ]]> + + + + + + + + + + + + + + + One row per gene per project, feeding the "Genetic Variation" section of the gene + record. Replaces six attributes retired from geneRecord.xml (total_hts_snps, + hts_nonsynonymous_snps, hts_synonymous_snps, hts_noncoding_snps, + hts_stop_codon_snps, hts_nonsyn_syn_ratio), which are already commented out on + master along with the TranscriptAttributes_p psql that fed them. + + Mirrors Model/lib/psql/webready/orgSpecific/GeneVariationSummary_p.psql. Any edit + here MUST be mirrored there, and vice versa. Full design, validation evidence, and + the biologist-facing help text: + docs/superpowers/specs/2026-08-07-gene-variation-summary-design.md + + THREE THINGS THAT LOOK LIKE STYLE BUT ARE LOAD-BEARING: + + 1. TWO GRAINS. Display counts are per GENE (unioned across transcripts, + most-severe-wins). The pi statistics are confined to the REPRESENTATIVE + longest-CDS transcript, whose id is stored in rep_transcript_source_id. piN/piS + is defined for one CDS; counting variants across all transcripts while + normalizing by one transcript's site counts would let a variant in a + transcript-specific exon enter the numerator while that exon's sites never + enter the denominator. Only 1.2% of pfal genes have >1 transcript today, but + splicing annotation density is an annotation property, not a constant. + + 2. SUPPRESS, NEVER DEGRADE. Every frequency-derived statistic accumulates only over + loci clearing an allele floor (pi >= 4, common/very-common >= 20, rare/singleton + >= 100) and is NULL below it, never 0. Sample size is a PER-LOCUS property: + min(called_strain_count) is 1 in all three loaded organisms and 35% of pfal loci + have fewer than 100 alleles. An organism with one sample plus a reference has 2 alleles, + where MAF can only be 0.5, so pi would compute to a rescaled variant density + carrying no frequency information. Rendering 0.00 there would reproduce the + exact defect of the retired hts_nonsyn_syn_ratio, which reported 0 for genes + with no synonymous sites and so displayed maximal signal as minimal. The + n_loci_* columns publish how many loci actually contributed. + + 3. PLOIDY IS DERIVED, NEVER HARDCODED (total_ploidy_count / called_strain_count). + Measured: pfal 1.01, tbru 2.27, afum 2.01 - afum being a HAPLOID fungus called + as diploid, which inflates its allele denominators 2x. A hardcoded lookup would + encode that calling bug as truth; a measured value self-corrects when the + upstream calling is fixed. + + MAF is a true ALLELE frequency (verified against pfal to within 2e-5 using + total_ploidy_count as the denominator), so 2p(1-p) is valid at any ploidy. But + *_minor_allele_strain_count is a STRAIN count - different units - so "singleton" is + defined on allele copies, round(maf * total_ploidy_count) = 1. + + Source is apidb.VariationEffect ALONE: at gene x locus grain + VariationTranscriptProduct contributes 0 pairs that VariationEffect lacks + (2,905,385 vs 1,691,462, 0 product-only), so the defensive UNION in + VariationAttributes is dead code at this grain. + + Site counts are Nei-Gojobori (1986), derived inline from the genetic code rather + than hardcoded, so the code table is the single source of truth. The pooled + synonymous-site fraction in pfal is 17.49%, NOT the textbook ~25% - an AT-bias + effect worth 1.43x on every gene. Without this normalization the median piN/piS is + 2.0, implying genome-wide positive selection; with it the median is 0.512, the + expected purifying-selection signature. + + COST NOTE: the codon work is streamed per-CDS via LATERAL rather than expanded flat + (see the gene_sites CTE). Measured over all 63,082 genes in unidb_shu_a it is 31 + seconds, so it is NOT the expensive step of this build - the VariationEffect + aggregation is. Do not "optimize" it into a flat expansion or a temp table; that was + measured 40% slower with a much larger peak footprint. + + + + + + + CTA), CGA 1.333. + nbr AS ( + SELECT c.codon, c.aa, p.pos, b.base, + overlay(c.codon placing b.base from p.pos for 1) AS mutated + FROM code c + CROSS JOIN generate_series(1,3) AS p(pos) + CROSS JOIN (VALUES ('A'),('C'),('G'),('T')) AS b(base) + WHERE c.aa <> '*' AND substr(c.codon, p.pos, 1) <> b.base + ), + posf AS ( + SELECT n.codon, n.pos, count(*) FILTER (WHERE m.aa = n.aa)::numeric / 3 AS f + FROM nbr n JOIN code m ON m.codon = n.mutated + GROUP BY 1,2 + ), + codon_sites AS ( + SELECT codon, sum(f) AS syn_sites, 3 - sum(f) AS nonsyn_sites + FROM posf GROUP BY codon + ), + -- representative transcript: longest CDS, deterministic tiebreak + rep AS ( + SELECT DISTINCT ON (gene_source_id, project_id) + gene_source_id, project_id, org_abbrev, organism, taxon_id, chromosome, + gene_na_feature_id, transcript_source_id, + na_feature_id AS rep_na_feature_id, + (gene_end_max - gene_start_min + 1) AS gene_length, cds_length + FROM TranscriptAttributes + ORDER BY gene_source_id, project_id, cds_length DESC NULLS LAST, + transcript_source_id + ), + -- per-gene site counts from the representative CDS. The length filter guards + -- against out-of-frame CDS rows, which would otherwise yield garbage codons + -- (477 of 63,765 rows genome-wide, none in the three loaded organisms). + -- LATERAL, not a flat expansion: postgres streams each CDS's codons through the + -- aggregate instead of materializing one row per codon across the whole genome. + -- Measured on unidb_shu_a over all 63,082 genes: 31s vs 52s for the flat form, + -- identical results (0 disagreements), and a far smaller peak footprint. A + -- PL/pgSQL loop would achieve the same but forces procedural code into both this + -- file and the webready mirror; this stays declarative, so the two copies remain + -- textually comparable. + gene_sites AS ( + SELECT r.gene_source_id, r.project_id, s.syn_sites, s.nonsyn_sites + FROM rep r + JOIN webready.CodingSequence_p cs + ON cs.source_id = r.transcript_source_id + AND cs.project_id = r.project_id + CROSS JOIN LATERAL ( + SELECT sum(k.syn_sites) AS syn_sites, + sum(k.nonsyn_sites) AS nonsyn_sites + FROM regexp_matches(upper(cs.sequence), '.{3}', 'g') AS m(arr) + JOIN codon_sites k ON k.codon = m.arr[1] + ) s + WHERE length(cs.sequence) % 3 = 0 AND length(cs.sequence) >= 6 + ), + sev AS ( + SELECT e.source, t.gene_source_id, t.project_id, e.na_feature_id, + e.sequence_source_id, e.location, + CASE e.effect + WHEN 'frameshift_variant' THEN 1 + WHEN 'stop_gained' THEN 2 + WHEN 'stop_lost' THEN 2 + WHEN 'start_lost' THEN 2 + WHEN 'splice_acceptor_variant' THEN 3 + WHEN 'splice_donor_variant' THEN 3 + WHEN 'conservative_inframe_deletion' THEN 4 + WHEN 'disruptive_inframe_deletion' THEN 4 + WHEN 'conservative_inframe_insertion' THEN 4 + WHEN 'disruptive_inframe_insertion' THEN 4 + WHEN 'inframe_deletion_unnormalized' THEN 4 + WHEN 'inframe_insertion_unnormalized' THEN 4 + WHEN 'missense_variant' THEN 5 + WHEN 'splice_region_variant' THEN 6 + WHEN 'synonymous_variant' THEN 7 + WHEN 'stop_retained_variant' THEN 7 + WHEN 'start_retained_variant' THEN 7 + WHEN '5_prime_UTR_variant' THEN 8 + WHEN '3_prime_UTR_variant' THEN 8 + WHEN '5_prime_UTR_premature_start_codon_gain_variant' THEN 8 + WHEN 'non_coding_transcript_exon_variant' THEN 9 + WHEN 'non_coding_transcript_variant' THEN 9 + WHEN 'intron_variant' THEN 10 + ELSE 11 + END AS sev, + CASE e.impact WHEN 'HIGH' THEN 4 WHEN 'MODERATE' THEN 3 + WHEN 'LOW' THEN 2 WHEN 'MODIFIER' THEN 1 END AS imp + FROM apidb.VariationEffect e + JOIN TranscriptAttributes t ON t.na_feature_id = e.na_feature_id + ), + vf AS ( + SELECT sequence_source_id, location, variant_type, is_coding, call_rate, + called_strain_count, het_strain_count, indel_frame_effect, + total_ploidy_count AS n_alleles, + nullif(greatest(coalesce(snp_minor_allele_frequency,0), + coalesce(indel_minor_allele_frequency,0)),0) AS maf + FROM apidb.VariationFeature + ), + gene_locus AS ( + SELECT source, gene_source_id, project_id, sequence_source_id, location, + min(sev) AS sev, max(imp) AS imp + FROM sev GROUP BY 1,2,3,4,5 + ), + gl AS ( + SELECT g.*, v.variant_type, v.is_coding, v.call_rate, v.called_strain_count, + v.het_strain_count, v.indel_frame_effect, v.n_alleles, v.maf, + round(v.maf * v.n_alleles) AS minor_copies + FROM gene_locus g JOIN vf v + ON v.sequence_source_id = g.sequence_source_id AND v.location = g.location + ), + snp_agg AS ( + SELECT gene_source_id, project_id, + count(*) AS total_variants, + count(*) FILTER (WHERE is_coding=1) AS n_coding_loci, + count(*) FILTER (WHERE variant_type='SNV') AS n_snv, + count(*) FILTER (WHERE variant_type='INDEL') AS n_indel, + count(*) FILTER (WHERE variant_type='MIXED') AS n_mixed, + count(*) FILTER (WHERE sev=1) AS n_frameshift, + count(*) FILTER (WHERE sev=2) AS n_nonsense, + count(*) FILTER (WHERE sev=3) AS n_splice_disruptive, + count(*) FILTER (WHERE sev=4) AS n_inframe_indel, + count(*) FILTER (WHERE sev=5) AS n_missense, + count(*) FILTER (WHERE sev=6) AS n_splice_region, + count(*) FILTER (WHERE sev=7) AS n_synonymous, + count(*) FILTER (WHERE sev=8) AS n_utr, + count(*) FILTER (WHERE sev=9) AS n_noncoding_exon, + count(*) FILTER (WHERE sev=10) AS n_intron, + count(*) FILTER (WHERE sev=11) AS n_other, + count(*) FILTER (WHERE sev<=3) AS n_lof, + count(*) FILTER (WHERE indel_frame_effect='frameshift') AS n_indel_frameshift, + count(*) FILTER (WHERE imp=4) AS n_impact_high, + count(*) FILTER (WHERE imp=3) AS n_impact_moderate, + count(*) FILTER (WHERE imp=2) AS n_impact_low, + count(*) FILTER (WHERE imp=1) AS n_impact_modifier, + CASE max(imp) WHEN 4 THEN 'HIGH' WHEN 3 THEN 'MODERATE' + WHEN 2 THEN 'LOW' WHEN 1 THEN 'MODIFIER' END AS most_severe_impact, + max(called_strain_count) AS max_called_strain_count, + round(percentile_cont(0.5) WITHIN GROUP (ORDER BY called_strain_count)::numeric,0) + AS median_called_strain_count, + max(n_alleles) AS max_alleles, + round(avg(n_alleles/nullif(called_strain_count,0))::numeric,2) AS effective_ploidy, + round(avg(call_rate)::numeric,3) AS avg_call_rate, + round(min(call_rate)::numeric,3) AS min_call_rate, + count(*) FILTER (WHERE call_rate < 0.5) AS n_low_call_rate, + count(*) FILTER (WHERE het_strain_count > 0) AS n_het_loci, + count(*) FILTER (WHERE n_alleles >= 4) AS n_loci_pi, + count(*) FILTER (WHERE n_alleles >= 20) AS n_loci_freq20, + count(*) FILTER (WHERE n_alleles >= 100) AS n_loci_freq100, + count(*) FILTER (WHERE n_alleles >= 20 AND maf > 0.05) AS n_common_raw, + count(*) FILTER (WHERE n_alleles >= 20 AND maf > 0.20) AS n_very_common_raw, + count(*) FILTER (WHERE n_alleles >= 20 AND maf > 0.05 AND sev=5) AS n_missense_common_raw, + count(*) FILTER (WHERE n_alleles >= 20 AND maf > 0.05 AND sev=7) AS n_synonymous_common_raw, + count(*) FILTER (WHERE n_alleles >= 20 AND maf > 0.05 AND sev<=3) AS n_lof_common_raw, + max(maf) FILTER (WHERE n_alleles >= 20) AS max_maf_raw, + count(*) FILTER (WHERE n_alleles >= 100 AND maf <= 0.01) AS n_rare_raw, + count(*) FILTER (WHERE n_alleles >= 100 AND minor_copies = 1) AS n_singleton_raw + FROM gl WHERE source='snpeff' GROUP BY 1,2 + ), + pc_agg AS ( + SELECT gene_source_id, project_id, + count(*) AS pc_total_coding_variants, + count(*) FILTER (WHERE sev=5) AS pc_n_missense, + count(*) FILTER (WHERE sev=7) AS pc_n_synonymous, + count(*) FILTER (WHERE sev<=3) AS pc_n_lof, + count(*) FILTER (WHERE sev=11) AS pc_n_unclassified, + count(*) FILTER (WHERE n_alleles >= 20 AND maf > 0.05 AND sev=5) AS pc_n_missense_common_raw, + count(*) FILTER (WHERE n_alleles >= 20 AND maf > 0.05 AND sev<=3) AS pc_n_lof_common_raw + FROM gl WHERE source='product_call' GROUP BY 1,2 + ), + tx_locus AS ( + SELECT s.source, r.gene_source_id, r.project_id, + s.sequence_source_id, s.location, min(s.sev) AS sev + FROM sev s JOIN rep r ON r.rep_na_feature_id = s.na_feature_id + GROUP BY 1,2,3,4,5 + ), + -- unbiased heterozygosity: (n/(n-1)) * 2p(1-p). The correction is ~0.4% at + -- pfal's 236 alleles but ~9% at tbru's 12, and 100% at the 2-allele floor - + -- which is why the >= 4 filter below exists rather than trusting the estimator. + tx_pi AS ( + SELECT t.source, t.gene_source_id, t.project_id, + sum(CASE WHEN t.sev=5 THEN (v.n_alleles/(v.n_alleles-1))*2*v.maf*(1-v.maf) END) AS pi_sum_nonsyn, + sum(CASE WHEN t.sev=7 THEN (v.n_alleles/(v.n_alleles-1))*2*v.maf*(1-v.maf) END) AS pi_sum_syn, + sum((v.n_alleles/(v.n_alleles-1))*2*v.maf*(1-v.maf)) AS pi_sum_all, + count(*) FILTER (WHERE t.sev=5) AS tx_n_missense, + count(*) FILTER (WHERE t.sev=7) AS tx_n_synonymous, + count(*) AS tx_n_loci_pi + FROM tx_locus t JOIN vf v + ON v.sequence_source_id = t.sequence_source_id AND v.location = t.location + WHERE v.n_alleles >= 4 AND v.maf IS NOT NULL + GROUP BY 1,2,3 + ) + SELECT + r.project_id, r.org_abbrev, r.organism, r.taxon_id, + r.gene_source_id, r.gene_na_feature_id, r.chromosome, + r.gene_length, r.cds_length, + r.transcript_source_id AS rep_transcript_source_id, + gs.syn_sites, gs.nonsyn_sites, + round((gs.syn_sites/nullif(gs.syn_sites+gs.nonsyn_sites,0))::numeric,4) AS syn_site_fraction, + a.max_called_strain_count, a.median_called_strain_count, a.max_alleles, + a.effective_ploidy, a.avg_call_rate, a.min_call_rate, a.n_low_call_rate, + a.n_het_loci, + round((a.n_het_loci::numeric/nullif(a.total_variants,0)),3) AS prop_het_loci, + a.n_loci_pi, a.n_loci_freq20, a.n_loci_freq100, + a.total_variants, a.n_coding_loci, a.n_snv, a.n_indel, a.n_mixed, + round((1000.0*a.total_variants/nullif(r.gene_length,0)),2) AS variants_per_kb, + a.n_frameshift, a.n_nonsense, a.n_splice_disruptive, a.n_inframe_indel, + a.n_missense, a.n_splice_region, a.n_synonymous, a.n_utr, + a.n_noncoding_exon, a.n_intron, a.n_other, + a.n_lof, a.n_indel_frameshift, + a.n_impact_high, a.n_impact_moderate, a.n_impact_low, a.n_impact_modifier, + a.most_severe_impact, + p.pc_total_coding_variants, p.pc_n_missense, p.pc_n_synonymous, + p.pc_n_lof, p.pc_n_unclassified, + -- frequency bins: NULL, not 0, when no locus cleared the floor + CASE WHEN a.n_loci_freq20 > 0 THEN a.n_common_raw END AS n_common, + CASE WHEN a.n_loci_freq20 > 0 THEN a.n_very_common_raw END AS n_very_common, + CASE WHEN a.n_loci_freq20 > 0 THEN a.n_missense_common_raw END AS n_missense_common, + CASE WHEN a.n_loci_freq20 > 0 THEN a.n_synonymous_common_raw END AS n_synonymous_common, + CASE WHEN a.n_loci_freq20 > 0 THEN a.n_lof_common_raw END AS n_lof_common, + CASE WHEN a.n_loci_freq20 > 0 THEN p.pc_n_missense_common_raw END AS pc_n_missense_common, + CASE WHEN a.n_loci_freq20 > 0 THEN p.pc_n_lof_common_raw END AS pc_n_lof_common, + CASE WHEN a.n_loci_freq20 > 0 THEN round(a.max_maf_raw::numeric,4) END AS max_minor_allele_frequency, + CASE WHEN a.n_loci_freq100 > 0 THEN a.n_rare_raw END AS n_rare, + CASE WHEN a.n_loci_freq100 > 0 THEN a.n_singleton_raw END AS n_singleton, + -- count ratios: a denominator under 5 synonymous loci is noise. NOT a + -- selection statistic - pfal's median count ratio is 2.22 while its median + -- site-normalized piN/piS is 0.512. Label it as a raw count ratio on the page. + CASE WHEN a.n_synonymous >= 5 + THEN round((a.n_missense::numeric/a.n_synonymous),2) END AS nonsyn_syn_ratio_snpeff, + CASE WHEN p.pc_n_synonymous >= 5 + THEN round((p.pc_n_missense::numeric/p.pc_n_synonymous),2) END AS nonsyn_syn_ratio_product_call, + round((se.pi_sum_nonsyn/nullif(gs.nonsyn_sites,0))::numeric,6) AS pi_nonsyn_snpeff, + round((se.pi_sum_syn /nullif(gs.syn_sites,0))::numeric,6) AS pi_syn_snpeff, + -- GUARDED twin: for sorting, searching and filtering, where a gene with 2 + -- synonymous sites would otherwise dominate a descending sort. + CASE WHEN se.tx_n_synonymous >= 5 THEN + round(((se.pi_sum_nonsyn/nullif(gs.nonsyn_sites,0)) + /nullif(se.pi_sum_syn/nullif(gs.syn_sites,0),0))::numeric,3) END AS pi_n_pi_s_snpeff, + -- UNGATED twin: for DISPLAY, rendered next to tx_n_synonymous_snpeff so the + -- reader can discount a thin denominator themselves ("10.32, from 4 synonymous + -- sites"). Needed because the guard blanks 57.5% of pfal genes, including + -- precisely the most-searched ones - strongly selected genes accumulate few + -- SYNONYMOUS variants, so AMA1 (4 sites), PfCRT (2) and Kelch13 (3) all + -- suppress while MSP1 (28) survives. A blank reads as "no data". + -- + -- This column cannot be dropped in favour of the model dividing + -- pi_nonsyn_snpeff by pi_syn_snpeff: those are stored rounded to 6 decimals + -- and pi is ~1e-3, so a reconstructed ratio drifts up to 0.048 from this one + -- (9% of genes differ). Two code paths must not yield two different numbers + -- for the same statistic. + round(((se.pi_sum_nonsyn/nullif(gs.nonsyn_sites,0)) + /nullif(se.pi_sum_syn/nullif(gs.syn_sites,0),0))::numeric,3) AS pi_n_pi_s_snpeff_ungated, + round((pp.pi_sum_nonsyn/nullif(gs.nonsyn_sites,0))::numeric,6) AS pi_nonsyn_product_call, + round((pp.pi_sum_syn /nullif(gs.syn_sites,0))::numeric,6) AS pi_syn_product_call, + CASE WHEN pp.tx_n_synonymous >= 5 THEN + round(((pp.pi_sum_nonsyn/nullif(gs.nonsyn_sites,0)) + /nullif(pp.pi_sum_syn/nullif(gs.syn_sites,0),0))::numeric,3) END AS pi_n_pi_s_product_call, + round(((pp.pi_sum_nonsyn/nullif(gs.nonsyn_sites,0)) + /nullif(pp.pi_sum_syn/nullif(gs.syn_sites,0),0))::numeric,3) AS pi_n_pi_s_product_call_ungated, + round((se.pi_sum_all/nullif(gs.syn_sites+gs.nonsyn_sites,0))::numeric,6) AS pi_per_site_cds, + se.tx_n_missense AS tx_n_missense_snpeff, + se.tx_n_synonymous AS tx_n_synonymous_snpeff, + se.tx_n_loci_pi AS tx_n_loci_pi_snpeff, + pp.tx_n_missense AS tx_n_missense_product_call, + pp.tx_n_synonymous AS tx_n_synonymous_product_call + FROM rep r + JOIN snp_agg a ON a.gene_source_id=r.gene_source_id AND a.project_id=r.project_id + LEFT JOIN pc_agg p ON p.gene_source_id=r.gene_source_id AND p.project_id=r.project_id + LEFT JOIN gene_sites gs ON gs.gene_source_id=r.gene_source_id AND gs.project_id=r.project_id + LEFT JOIN tx_pi se ON se.gene_source_id=r.gene_source_id AND se.project_id=r.project_id + AND se.source='snpeff' + LEFT JOIN tx_pi pp ON pp.gene_source_id=r.gene_source_id AND pp.project_id=r.project_id + AND pp.source='product_call' + ]]> + + + + + + + + + @@ -3376,6 +3857,131 @@ create index Organism_projectId_idx&1 ON OrganismAttributes&1 (project_id, sourc ON DatasetVariable&1 (dataset_id) ]]> - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/superpowers/plans/2026-07-30-variation-record.md b/docs/superpowers/plans/2026-07-30-variation-record.md new file mode 100644 index 000000000..229ee4cbc --- /dev/null +++ b/docs/superpowers/plans/2026-07-30-variation-record.md @@ -0,0 +1,2253 @@ +# Variation Record Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Build a new WDK `variation` record — one record per variant locus — with the attributes, tuning table, and record-page tables specified in `docs/superpowers/specs/2026-07-30-variation-record-design.md`. + +**Architecture:** A thin `apidbtuning.VariationAttributes` tuning table supplies derived, aggregated, and join-requiring columns; `apidb.VariationFeature` is queried directly for intrinsic per-locus facts (it is already one row per locus with a unique `source_id`). Two record-page tables read `apidb.VariationTranscriptProduct` and `apidb.VariationEffect`. Work proceeds as a vertical slice: a minimal buildable record first (Task 4), then attributes and tables added incrementally, each verified by a real build against a live instance. + +**Tech Stack:** WDK model XML, EuPathDB tuning manager XML, PostgreSQL, `wb` build wrapper via `bin/veup-build.sh`, `wdkQuery` for SQL introspection, WDK REST service for verification. + +--- + +## Context you need before starting + +> ### Progress +> +> **All tasks (0-14) are complete.** Reconciled against the commit log on +> `dnaseq-merge-experiments` on 2026-08-05; the block below had been stale since Task 4. +> +> | task | commit(s) | +> |---|---| +> | 0-1 preconditions, psql SELECT check | verification only, no commit | +> | 2 tuning table definition | `9ff6094` | +> | 3 build and verify the tuning table | see the note below — superseded by a tuning run | +> | 4 minimal buildable record | `9bcf147` | +> | 5 classification attributes | `48f13bc` | +> | 6 SNP and Indel allele sections | `d35e557`, `51cb41b` (assemble allele strings in SQL) | +> | 7 strain and call statistics | `af9b2aa`, `e0bb106` (rename to "Called Strain Count") | +> | 8 gene linkage, effect rollups, collapsed columns | `d8ccd76`, `e3571e2` (impact sort, MAF help) | +> | 9 record overview and default summary | `f9264c0`, `d636b44` (label allele rows by class) | +> | 10 TranscriptProducts table | `dd4efa6`, `9cc505b` (strain_count help text) | +> | 11 PredictedEffects table | `b19c82d` | +> | 12 category ontology placement | `cf0dd13`, `19b6527` (ontology parenting) | +> | 13 final end-to-end verification | `19b6527` — the four review findings it fixes are that pass's output; there is no separate report doc | +> | 14 flip the stub to the real tuning table | `2723cda` | +> +> Task 4's original detail, kept because the defects it surfaced are still worth knowing: +> the record built green with 13 attributes registered and +> `/plasmo.jbrestel/app/record/variation/Variant_Pf3D7_01_v3_29514` rendering with every +> service call 200 and all error logs silent. Three plan defects came out of it — the +> app/service base URLs both include `/plasmo.jbrestel`, an empty `querySet` is invalid so +> `variationTableQueries.xml` moved to Task 10, and the snp imports sit inside a comment +> block that would have swallowed the new imports. +> +> Tuning table as verified at Task 3: **4,390,908 rows**, exactly matching +> `apidb.VariationFeature`, built in 66s with three indexes and `GRANT SELECT TO gus_r`. +> `gene_ids` populated for 2,879,337 loci, `most_severe_impact_snpeff` for 4,390,895, +> `most_severe_impact_product_call` for 1,690,908, `collapsed_allele` for all 4,390,908, +> 3 projects, **25,545 multi-gene loci** — matching the spec's figure exactly. All three +> spot-check loci correct, including `A>C; A>AC` for the MIXED locus. +> +> ### ✓ The developer-schema stub is gone (was: the model reads a stub) +> +> Resolved. This warning described Tasks 4-13 writing `jbrestel.VariationAttributes` into +> the query XML in six places, because `tuningManager` was not installed on the dev +> instance when the plan was written. +> +> Both halves have since been settled, verified 2026-08-05: +> +> - **Model:** Task 14 (`2723cda`) flipped all six references to +> `ApidbTuning.VariationAttributes`. No `jbrestel` reference remains anywhere in the +> variation model XML. +> - **Database:** a tuning run (Jenkins) built the real thing on `unidb_shu_a` — +> `apidbtuning.variationattributes1121` (1688 MB, 4,390,908 rows) plus the +> `apidbtuning.variationattributes` view over it, `SELECT` granted to `gus_r`. All 17 +> `va.*` columns the model reads resolve against that view. `jbrestel.VariationAttributes` +> no longer exists. +> +> So the merge blocker is clear, and the flip does **not** cost buildability on the dev +> instance the way this plan originally assumed it would. +> +> The Task 4-11 bodies below still show `jbrestel.VariationAttributes` in their SQL +> snippets, deliberately: that is what was executed at the time, and rewriting them would +> describe a history that never happened. **Do not copy those snippets forward** — the +> committed XML is the current truth. Read them as a record, not as instructions. + +**Two repos are involved:** + +| repo | path | role | +|---|---|---| +| `ApiCommonModel` | `~/workspaces/plasmodb/ApiCommonModel` | all edits land here; branch `dnaseq-merge-experiments` | +| `agentic-veupath-dev` | `~/workspaces/agentic-veupath-dev` | control plane — run builds from here | + +**Concrete instance values** (resolved from `profiles/plasmodb.yml` + `profiles/identity.yml`): + +| value | | +|---|---| +| ssh host | `cedar` | +| docroot | `/var/www/jbrestel.plasmodb.org/project_home` | +| setenv | `/var/www/jbrestel.plasmodb.org/etc/setenv` | +| app URL | `https://jbrestel.plasmodb.org/plasmo.jbrestel/app` | +| service base | `/plasmo.jbrestel/service` (alias `/a/service`) — **not** `/service` | +| GUS project | `PlasmoDB` | +| local appDb (psql) | `unidb_shu_a` on `localhost:5432` | + +**Commands you will use repeatedly:** + +```bash +# Build the WDK model (questions, queries, records). Run from the harness repo. +cd ~/workspaces/agentic-veupath-dev && bin/veup-build.sh plasmodb wb model + +# Rebuild the category OWL *and* the model. Required for any individuals.txt change. +cd ~/workspaces/agentic-veupath-dev && bin/veup-build.sh plasmodb wb ontology + +# Render a WDK query's assembled SQL without executing it (always safe). +ssh cedar 'bash -lc "source /var/www/jbrestel.plasmodb.org/etc/setenv && \ + wdkQuery -model PlasmoDB -query -showQuery"' + +# Read remote logs by page-load delta. +cd ~/workspaces/agentic-veupath-dev && bin/veup-logs.sh plasmodb mark     ` of `withNgsSNPsTree` (line 589), inside the `organismVQ` querySet: + +```xml + + + + + + + + + + + + + +``` + +Note the column element is `parentTerm` (camel case) — that is what `withNgsSNPsTree` declares, and WDK matches it case-insensitively to the SQL's `parentterm`. + +- [ ] **Step 4: Verify the XML still parses** + +```bash +python3 -c "import xml.etree.ElementTree as T; T.parse('$HOME/workspaces/plasmodb/ApiCommonModel/Model/lib/wdk/model/questions/params/organismParams.xml'); print('parses')" +``` + +Expected: `parses`. (This catches an unbalanced tag in seconds instead of at the end of a five-minute remote build.) + +- [ ] **Step 5: Commit** + +```bash +cd ~/workspaces/plasmodb/ApiCommonModel +git add Model/lib/wdk/model/questions/params/organismParams.xml +git commit -m "Add organismVQ.withVariationsTree for the variation searches + +The snp original (withNgsSNPsTree) reads apidbtuning.snpstrains, which does +not exist in this build, so this is a rewrite. Filters on apidb.datasource's +declared type='isolates'/subtype='Dna_Seq' -- the 10 dnaseq experiment +datasets -- rather than a name string convention. + +internal is the taxon name, not an abbreviation: the HSSS plugin resolves it +through sres.TaxonName to get name_for_filenames for the webservices path." +``` + +--- + +### Task 3: `eda_sample_table_suffix` — the hidden dependent param + +The one non-obvious piece of the design. The organism param's internal value **must** be the taxon name (the plugin needs it); the EDA queries need the study+entity abbreviation to build table names. Two identities, one dropdown — resolved with a hidden param that carries the second identity. Design §2 and §4.2. + +**Files:** +- Modify: `Model/lib/wdk/model/questions/params/variationParams.xml` (new `querySet VariationVQ` + one param) + +- [ ] **Step 1: Run the vocabulary SQL — it must return exactly one row** + +```bash +psql -h localhost -p 5432 -d unidb_shu_a -c " +SELECT DISTINCT + s.internal_abbrev || '_' || lower(e.internal_abbrev) AS internal, + s.internal_abbrev || '_' || lower(e.internal_abbrev) AS term +FROM apidb.datasource ds +JOIN apidb.organism o ON o.taxon_id = ds.taxon_id +JOIN sres.taxonname tn ON tn.taxon_id = o.taxon_id AND tn.name_class = 'scientific name' +JOIN sres.externaldatabase ed ON ed.name = ds.name +JOIN sres.externaldatabaserelease edr ON edr.external_database_id = ed.external_database_id +JOIN eda.studyexternaldatabaserelease sedr ON sedr.external_database_release_id = edr.external_database_release_id +JOIN eda.study s ON s.study_id = sedr.study_id +JOIN eda.entitytypegraph e ON e.study_id = s.study_id +WHERE ds.type = 'isolates' AND ds.subtype = 'Dna_Seq' + AND tn.name = 'Plasmodium falciparum 3D7' + AND s.internal_abbrev IS NOT NULL" +``` + +Expected: exactly **one row**, `s3be28bbe14_sample`. More than one row means the tables interpolated downstream would be ambiguous — stop; the joins need narrowing before anything else is written. + +- [ ] **Step 2: Confirm the tables that name implies actually exist** + +```bash +psql -h localhost -p 5432 -d unidb_shu_a -tAc " +SELECT to_regclass('eda.attributevalue_s3be28bbe14_sample'), + to_regclass('eda.attributegraph_s3be28bbe14_sample')" +``` + +Expected: both non-null. This is the check that a *looked-up* abbreviation buys you and a recomputed hash does not — see design §3. + +- [ ] **Step 3: Add the `VariationVQ` querySet with this one query** + +In `variationParams.xml`, after the closing `` and before ``: + +```xml + + + + + + + + + + + + + + +``` + +`noTranslation="true"` on the organism `paramRef` makes WDK pass the param's **term** rather than its internal — for a selected leaf these are the same taxon name, and it is what `SnpVQ` did. The SQL supplies the quotes, so the param must not also be quoted. + +- [ ] **Step 4: Add the param itself** + +Inside the existing `variationParams` paramSet, after the `variation_id` `datasetParam`: + +```xml + + + + + + Derived from the selected organism. Not user-visible. + +``` + +- [ ] **Step 5: Verify the XML parses** + +```bash +python3 -c "import xml.etree.ElementTree as T; T.parse('$HOME/workspaces/plasmodb/ApiCommonModel/Model/lib/wdk/model/questions/params/variationParams.xml'); print('parses')" +``` + +Expected: `parses`. + +- [ ] **Step 6: Commit** + +```bash +cd ~/workspaces/plasmodb/ApiCommonModel +git add Model/lib/wdk/model/questions/params/variationParams.xml +git commit -m "Add hidden eda_sample_table_suffix param and VariationVQ querySet + +The organism param's internal value must be the taxon name -- the HSSS plugin +resolves it through sres.TaxonName to build the webservices path. The EDA +filter queries need the study+entity abbreviation to name per-study tables. +Two identities for one dropdown, so the second travels in a hidden dependent +param and gets interpolated into the table name. + +Looked up rather than recomputed from the SHA-1 convention: a lookup fails +visibly (empty dropdown), a stale hash fails invisibly (missing relation, +much later, from inside a filterParam)." +``` + +--- + +### Task 4: `variation_sample_meta` — the EDA-driven samples filter + +**Files:** +- Modify: `Model/lib/wdk/model/questions/params/variationParams.xml` (two queries into `VariationVQ`, one `filterParam`) + +> **The param name `variation_sample_meta` is a contract, not a style choice.** +> `FindPolymorphismsPlugin.getStrainFilterParamName()` returns exactly this string and +> `FindPolymorphismsAbstractPlugin` lists it among the plugin's **required** params. Any +> other spelling is rejected at run time as a missing required parameter. Do not "improve" it. + +- [ ] **Step 1: Run the metadata SQL** + +```bash +psql -h localhost -p 5432 -d unidb_shu_a -c " +SELECT count(*) AS rows, + count(DISTINCT av.sample_stable_id) AS samples, + count(DISTINCT av.attribute_stable_id) AS attributes, + count(av.string_value) AS strings, + count(av.number_value) AS numbers, + count(av.date_value) AS dates +FROM eda.attributevalue_s3be28bbe14_sample av" +``` + +Expected: `3771 | 216 | 20 | 2013 | 1758 | 0`. Zero dates is fine — no date-typed filters will appear, which is this dataset, not a defect. + +- [ ] **Step 2: Run the ontology SQL and check WDK's two throw conditions** + +WDK's `OntologyItemNewFetcher.validateOntologyItems` throws if (a) any node names a parent that is not itself a node, or (b) any node with a NULL type has no children. Check both before writing XML: + +```bash +psql -h localhost -p 5432 -d unidb_shu_a -c " +WITH t AS ( + SELECT ag.stable_id AS ontology_term_name, + CASE WHEN ag.parent_stable_id IN + (SELECT stable_id FROM eda.attributegraph_s3be28bbe14_sample) + THEN ag.parent_stable_id + END AS parent_ontology_term_name, + CASE ag.data_type + WHEN 'string' THEN 'string' + WHEN 'number' THEN 'number' + WHEN 'integer' THEN 'number' + END AS type + FROM eda.attributegraph_s3be28bbe14_sample ag +) +SELECT (SELECT count(*) FROM t) AS nodes, + (SELECT count(*) FROM t WHERE parent_ontology_term_name IS NULL) AS roots, + (SELECT count(*) FROM t WHERE parent_ontology_term_name IS NOT NULL + AND parent_ontology_term_name NOT IN (SELECT ontology_term_name FROM t)) AS dangling, + (SELECT count(*) FROM t WHERE type IS NULL AND ontology_term_name NOT IN + (SELECT parent_ontology_term_name FROM t WHERE parent_ontology_term_name IS NOT NULL)) AS childless_branches, + (SELECT count(*) FROM t WHERE type IS NOT NULL) AS typed_leaves" +``` + +Expected: `27 | 7 | 0 | 0 | 20`. + +**`dangling` and `childless_branches` must both be 0.** They are what the `CASE` on the parent is for: EDA's attribute graph has **no row for the entity itself**, so seven category nodes declare `parent_stable_id = 'sample'` and nothing has `stable_id = 'sample'`. Mapping unresolvable parents to NULL makes WDK adopt them under its own synthetic master root. + +- [ ] **Step 3: Prove the naive version would in fact fail** + +Worth ten seconds, because the `CASE` looks like noise until you see this: + +```bash +psql -h localhost -p 5432 -d unidb_shu_a -tAc " +SELECT count(*) FROM eda.attributegraph_s3be28bbe14_sample ag +WHERE ag.parent_stable_id IS NOT NULL + AND ag.parent_stable_id NOT IN (SELECT stable_id FROM eda.attributegraph_s3be28bbe14_sample)" +``` + +Expected: `7`. Those are seven guaranteed `WdkModelException`s if you write `ag.parent_stable_id` unguarded. + +- [ ] **Step 4: Confirm the filter's internal values are usable by HSSS** + +The filter hands sample stable IDs to the plugin, which writes them to a strains file with `strains_are_names = 1`. They must be names HSSS knows: + +```bash +psql -h localhost -p 5432 -d unidb_shu_a -tAc " +SELECT count(DISTINCT sample_stable_id) FROM eda.attributevalue_s3be28bbe14_sample" \ +&& ssh cedar "cut -f2 /var/www/Common/apiSiteFilesMirror/webServices/PlasmoDB/build-71/Pfalciparum3D7/dnaseq/readFreq20/strainIdToName.dat | sort -u | wc -l" +``` + +Expected: `216` and `538`. EDA covers a subset of the strains HSSS knows, which is the safe direction: the filter cannot offer a strain HSSS has never heard of. (The strict-subset relation was verified when the design was written; the counts here are the cheap re-check.) + +- [ ] **Step 5: Add both queries to `VariationVQ`** + +After `EdaSampleTableSuffix`, inside the `VariationVQ` querySet: + +```xml + + + + + + + + + + + + + + + + + + + + + + + + + + + + +``` + +- [ ] **Step 6: Add the `filterParam`** + +Inside `variationParams`, after `eda_sample_table_suffix`: + +```xml + + + + + + + Select a set of samples whose genomic sequences will be compared. Use the + sample characteristics to narrow the group, or accept all samples for the + organism you chose. + + +``` + +- [ ] **Step 7: Verify the XML parses** + +```bash +python3 -c "import xml.etree.ElementTree as T; T.parse('$HOME/workspaces/plasmodb/ApiCommonModel/Model/lib/wdk/model/questions/params/variationParams.xml'); print('parses')" +``` + +Expected: `parses`. + +- [ ] **Step 8: Commit** + +```bash +cd ~/workspaces/plasmodb/ApiCommonModel +git add Model/lib/wdk/model/questions/params/variationParams.xml +git commit -m "Add EDA-driven variation_sample_meta filterParam and its two queries + +The samples filter moves from apidbTuning.Metadata to EDA's per-study tables, +named by interpolating the hidden suffix param. + +The CASE on parent_stable_id is load-bearing, not defensive: EDA's attribute +graph has no row for the entity itself, so seven category nodes point at a +parent 'sample' that does not exist as a node, and WDK's validateOntologyItems +throws on precisely that. Mapping unresolvable parents to NULL hands them to +WDK's synthetic master root. Verified 27 nodes -> 7 roots, 0 dangling, 0 +childless branches, 20 typed leaves. + +The param name is the plugin's required-parameter contract; do not rename it." +``` + +--- + +### Task 5: The four HSSS path and threshold params + +Copied out of `snpParams.xml`, **not** referenced there. Design §4.4: `snpParams.xml` is imported inside the commented-out snp block, so the `snpParams` paramSet is absent from the assembled model and every `paramRef` to it would fail model load. Uncommenting is not the fix either — that file references `SnpRecordClasses.SnpRecordClass`, also commented out. + +**Files:** +- Modify: `Model/lib/wdk/model/questions/params/variationParams.xml` + +- [ ] **Step 1: Confirm for yourself that `snpParams` is not in the assembled model** + +Do not take the plan's word for it, and do not use `grep` on `apiCommonModel.xml` — grep cannot tell a live import from a commented-out one: + +```bash +ssh cedar 'bash -lc "source /var/www/jbrestel.plasmodb.org/etc/setenv && wdkXml -model PlasmoDB"' > /tmp/wdkxml.txt +grep -c snpParams /tmp/wdkxml.txt +grep -oE "ParamSet: name='[^']*'" /tmp/wdkxml.txt | sort -u +``` + +Expected: `0`, and a list of ~22 paramSets that includes `variationParams` and does not include `snpParams`. That zero is the whole justification for copying rather than referencing. + +> **Grep the bare name, never `name="snpParams"`.** `wdkXml` prints attributes with **single** quotes (`name='snpParams'`), so a double-quoted pattern reports `0` whether or not the paramSet is there — a check that can only pass. Printing the full paramSet list is what makes the zero mean something. + +- [ ] **Step 2: Confirm the four read-frequency directories exist, since the internals must match them** + +```bash +ssh cedar 'ls /var/www/Common/apiSiteFilesMirror/webServices/PlasmoDB/build-71/Pfalciparum3D7/dnaseq/' +``` + +Expected: `readFreq20 readFreq40 readFreq60 readFreq80` (plus `bigwig` and `vcf`, which are out of scope). The four `ReadFrequencyPercent` internals below must be exactly `20`/`40`/`60`/`80`. + +- [ ] **Step 3: Add the four params** + +Inside `variationParams`, after `variation_sample_meta`: + +```xml + + + + + + + + + + dflt + @WEBSERVICEMIRROR@/PROJECT_GOES_HERE/build-%%buildNumber%% + + + + + + + + This parameter applies to the sequencing reads of individual samples and + defines a stringency for data supporting a variant call between a sample and + the reference genome (Organism). Each nucleotide position of each sample is + compared to the reference genome and a call is made if the portion of the + sample's aligned reads that support the variant is above the Read Frequency + Threshold (RFT). Find high quality haploid variants with 80% RFT or + heterozygous diploid/aneuploid variants with 40%. See the Description below + for more. + + + + 80% + 80 + + + 60% + 60 + + + 40% + 40 + + + 20% + 20 + + + + + + + This parameter applies to your group of samples. A variant can occur in any + number of samples in your group and the least frequent call across all + samples is the Minor Allele Frequency. A variant will be returned by the + search if the frequency of the minor allele is equal to or greater than your + Minor Allele Frequency. See the Description below the Get Answer button for + more. + + + \d\d? + + + + + This parameter applies to the selected set of aligned sample sequences. At + any given nucleotide position, some samples in your group may not have data + supporting a base call because the Read Frequency Threshold was not met or + fewer than our minimum of 5 reads aligned. 'Percent samples with a base call' + defines the fraction of the selected samples that must have a base call + before a variant is returned for that nucleotide position, based on the + remaining samples that do have data. See the Description below for more + information. + + + \d\d?|100 + +``` + +The param **names** stay as the originals (including `MinPercentIsolateCalls`) — the plugin reads them by name. Only prompts and help text move from "isolates" to "samples". + +- [ ] **Step 4: Verify the XML parses and the regexes survived escaping** + +```bash +python3 - <<'PY' +import xml.etree.ElementTree as T, os +p = os.path.expanduser('~/workspaces/plasmodb/ApiCommonModel/Model/lib/wdk/model/questions/params/variationParams.xml') +r = T.parse(p).getroot() +for sp in r.iter('stringParam'): + rx = sp.find('regex') + print(sp.get('name'), '->', repr(rx.text if rx is not None else None)) +PY +``` + +Expected exactly: +``` +MinPercentMinorAlleles -> '\\d\\d?' +MinPercentIsolateCalls -> '\\d\\d?|100' +``` +(Python shows a literal backslash as `\\`. If you see `\\\\d`, the backslashes got doubled — fix it; the regex would reject every input.) + +- [ ] **Step 5: Build, and prove the model loads with all five new params** + +This is the first remote build. It takes a few minutes. + +```bash +cd ~/workspaces/agentic-veupath-dev && bin/veup-build.sh plasmodb wb model +``` + +Expected: completes without a `WdkModelException`. If it fails, the message names the unresolved reference — that is the whole value of this step. + +Then prove the params are in the *assembled* model, not merely in a file: + +```bash +ssh cedar 'bash -lc "source /var/www/jbrestel.plasmodb.org/etc/setenv && wdkXml -model PlasmoDB"' \ + | grep -E "name='(eda_sample_table_suffix|variation_sample_meta|WebServicesPath|ReadFrequencyPercent|MinPercentMinorAlleles|MinPercentIsolateCalls)'" +``` + +Expected: one matching line per name, six in all, each prefixed with its Java param class (`FlatVocabParam`, `FilterParamNew`, `EnumParam`, `StringParam`). Note the **single** quotes — that is how `wdkXml` prints attributes. `wb model` is correct here — nothing has touched categorization yet; Task 8 is what forces `wb ontology`. + +- [ ] **Step 6: Commit** + +```bash +cd ~/workspaces/plasmodb/ApiCommonModel +git add Model/lib/wdk/model/questions/params/variationParams.xml +git commit -m "Copy the four HSSS path and threshold params into variationParams + +Not referenced from snpParams: that file is imported inside the commented-out +snp block, so the snpParams paramSet is absent from the assembled model +(wdkXml finds it nowhere) and every paramRef to it would fail model load. +Uncommenting the one import fails too -- the file references +SnpRecordClasses.SnpRecordClass, also commented out. The originals are +unimported dead code, so this is migration out of a dead file. + +Two deliberate departures: one ReadFrequencyPercent rather than the original's +two (they differed only in help text phrased for the two-group search), and +help text saying 'samples' to match the EDA vocabulary the filter is built +from. Param names are unchanged -- the plugin reads them by name." +``` + +--- + +### Task 6: The `processQuery` + +**Files:** +- Modify: `Model/lib/wdk/model/questions/queries/variationQueries.xml` + +- [ ] **Step 1: Confirm the plugin class and its required params before wiring to them** + +```bash +cd ~/workspaces/plasmodb/ApiCommonWebService +grep -rn "getStrainFilterParamName\|PARAM_ORGANISM\|REQUIRED" \ + WSFPlugin/src/main/java/org/apidb/apicomplexa/wsfplugin/highspeedsnpsearch/FindPolymorphismsPlugin.java \ + WSFPlugin/src/main/java/org/apidb/apicomplexa/wsfplugin/highspeedsnpsearch/FindPolymorphismsAbstractPlugin.java \ + | head -20 +``` + +Expected: `FindPolymorphismsPlugin` returns `"variation_sample_meta"` from `getStrainFilterParamName()`, and the abstract plugin lists that name among its required params. If the returned string is anything else, **stop** — Task 4's param name must match it, and the plan is stale. + +- [ ] **Step 2: Add the query** + +In `variationQueries.xml`, inside the `VariationsBy` querySet, after the `VariationBySourceId` `sqlQuery`: + +```xml + + + + + + + + The Organism defines the species identity of the samples and the genome + against which each sample's variants were called. After choosing an Organism, + the set of samples available for forming groups is limited to samples aligned + to your chosen Organism's genome. + The organism you choose determines the genome to which the + variants have been mapped. It also restricts the set of samples you may + choose, since variants are identified by aligning that sample's reads to this + genome. + + + + + + + + + + + + + + + +``` + +- [ ] **Step 3: Verify the XML parses** + +```bash +python3 -c "import xml.etree.ElementTree as T; T.parse('$HOME/workspaces/plasmodb/ApiCommonModel/Model/lib/wdk/model/questions/queries/variationQueries.xml'); print('parses')" +``` + +Expected: `parses`. + +- [ ] **Step 4: Commit** (no build yet — Task 7's question is what makes this query reachable, and one build covers both) + +```bash +cd ~/workspaces/plasmodb/ApiCommonModel +git add Model/lib/wdk/model/questions/queries/variationQueries.xml +git commit -m "Add VariationsByIsolateGroup process query + +Binds FindPolymorphismsPlugin to the new variation params. The wsColumn set is +dictated by the plugin, which throws unless its results file has exactly four +tab-separated columns; project_id comes from the plugin rather than the file. +organism quote=false is required for the dependent-param query." +``` + +--- + +### Task 7: The question + +**Files:** +- Modify: `Model/lib/wdk/model/questions/variationQuestions.xml` + +- [ ] **Step 1: Confirm the summary attributes exist on the record** + +An `attributesList` naming an attribute the record does not have fails model load: + +```bash +cd ~/workspaces/plasmodb/ApiCommonModel +grep -cE 'name="(variation_location|gene_ids|variant_type)"' Model/lib/wdk/model/records/variationRecords.xml +``` + +Expected: `3`. The other three summary columns (`PercentMinorAlleles`, `PercentIsolateCalls`, `Phenotype`) are the query's dynamic `wsColumn`s from Task 6 and will not appear in the record file — that is correct. + +- [ ] **Step 2: Add the question** + +In `variationQuestions.xml`, inside the `VariationQuestions` questionSet, after `VariationBySourceId`: + +```xml + + + + + + + + + + + + + +
    + + Each sample's sequencing reads are aligned to the reference genome + (Organism) and variants are recorded for each sample based on the Read + Frequency Threshold. Then, scanning variant locations across the group of + samples, variants are returned by the search if the Minor Allele Frequency + and the Percent samples with a base call are met. + +

    Organism: The Organism parameter defines the species of the + samples and the genome in which the variants are determined. Choosing an + Organism focuses the Samples parameter to the samples of that organism, + changing the subset available when forming your group.

    + +

    Samples: Sample sequences are accompanied by characteristics of + the sample -- where it was collected, the host, alignment statistics. By + default the group includes all samples from the Organism you chose; you may + narrow the group using those characteristics. At least two samples are + required, since polymorphism within a group of one is undefined.

    + +

    Read frequency threshold: An allele is called for a sample at a + location if that fraction of the sample's aligned reads support it. For + example, a sample with 10 reads at a location -- 6 A and 4 C -- is called A + at a threshold of 60% or less, and not called at 80%. This matters most for + diploid or aneuploid organisms, where heterozygous positions are expected + near 50%.

    + +

    Minor allele frequency: Among the qualifying calls at a location, + the minor allele frequency is the percent that are not the major allele. A + location is returned if that is at or above the value you specify. Use 0 to + find every variant location within the group.

    + +

    Percent samples with a base call: A location is only considered if + this fraction of your selected samples have a qualifying call there. With 20 + samples and a threshold of 75%, a location with fewer than 15 called samples + is ignored.

    + ]]> +     + + + + + + Display the histogram of the values of this attribute + int + + + + + Display the histogram of the values of this attribute + int + + + + + +     +``` + +> **The `dynamicAttributes` block is mandatory.** An earlier version of this plan omitted it and the build failed with `Summary attribute field [PercentMinorAlleles] defined in question [...] is invalid` — `attributesList` may not reference a `processQuery`'s `wsColumn`s until they are declared as dynamic attributes on the question. The three remaining HSSS variation searches will each need their own block. + +```bash +python3 -c "import xml.etree.ElementTree as T; T.parse('$HOME/workspaces/plasmodb/ApiCommonModel/Model/lib/wdk/model/questions/variationQuestions.xml'); print('parses')" +``` + +Expected: `parses`. + +- [ ] **Step 4: Build and prove the question is in the assembled model** + +```bash +cd ~/workspaces/agentic-veupath-dev && bin/veup-build.sh plasmodb wb model +``` + +Expected: completes without a `WdkModelException`. + +```bash +ssh cedar 'bash -lc "source /var/www/jbrestel.plasmodb.org/etc/setenv && wdkXml -model PlasmoDB"' \ + | grep -c "VariationsByIsolateGroup" +``` + +Expected: a non-zero count. Zero means the question is in the file but not in the model for PlasmoDB — check `includeProjects` on the enclosing questionSet. + +- [ ] **Step 5: Commit** + +```bash +cd ~/workspaces/plasmodb/ApiCommonModel +git add Model/lib/wdk/model/questions/variationQuestions.xml +git commit -m "Add the VariationsByIsolateGroup question + +Overrides attributesList, unlike VariationBySourceId: the plugin's dynamic +minor-allele-frequency and percent-called columns are the point of the search +and are not in the record's default summary. + +displayName says Samples rather than the snp original's Isolates, matching the +EDA vocabulary the filter is built from. noSummaryOnSingleRecord deliberately +unset -- a one-hit analytical result wants its context." +``` + +--- + +### Task 8: Category ontology placement + +**Files:** +- Modify: `Model/lib/wdk/ontology/individuals.txt` + +`individuals.txt` is tab-delimited with 14 columns **and load-bearing empty fields**, including a trailing tab. Do not hand-type the row — derive it from the `VariationBySourceId` row so the whitespace is exact. + +- [ ] **Step 1: Look at the row you are copying** + +```bash +cd ~/workspaces/plasmodb/ApiCommonModel +grep -n "VariationQuestions.VariationBySourceId" Model/lib/wdk/ontology/individuals.txt | cat -A +``` + +Expected: one match (line ~1158) showing `^I` between every field, `topic_0199` as the parent, `search` as the target type, `menu` and `webservice` at the end, and a trailing `^I` before `$`. + +- [ ] **Step 2: Append the new row by substitution, not by typing** + +```bash +cd ~/workspaces/plasmodb/ApiCommonModel +grep "VariationQuestions.VariationBySourceId" Model/lib/wdk/ontology/individuals.txt \ + | sed 's/VariationBySourceId/VariationsByIsolateGroup/g' \ + >> Model/lib/wdk/ontology/individuals.txt +``` + +- [ ] **Step 3: Verify field count and content** + +```bash +cd ~/workspaces/plasmodb/ApiCommonModel +tail -n 1 Model/lib/wdk/ontology/individuals.txt | cat -A +tail -n 1 Model/lib/wdk/ontology/individuals.txt | awk -F'\t' '{print NF" fields"}' +grep -c "VariationQuestions.VariationsByIsolateGroup" Model/lib/wdk/ontology/individuals.txt +``` + +Expected: the new row with `VariationsByIsolateGroup` in columns 1 and 6 and everything else identical to the `VariationBySourceId` row; the same field count that row has (compare with `grep VariationBySourceId ... | awk -F'\t' '{print NF}'` — they must match); and `1`. + +- [ ] **Step 4: Build with `wb ontology` — NOT `wb model`** + +```bash +cd ~/workspaces/agentic-veupath-dev && bin/veup-build.sh plasmodb wb ontology +``` + +Expected: completes. `wb ontology` regenerates `individuals.owl` → `categories_merged.owl`, which is what the app actually reads, and does the model build too. Run `wb model` here instead and the site keeps serving the previous tree with **no error anywhere** — the search exists in the model but is uncategorized and absent from the menu. + +- [ ] **Step 5: Prove the OWL actually contains it** + +```bash +ssh cedar 'grep -c "VariationsByIsolateGroup" /var/www/jbrestel.plasmodb.org/gus_home/lib/wdk/ontology/categories_merged.owl' +``` + +Expected: non-zero. This is the server-side answer to "is it categorized where I intended" and needs no browser. + +- [ ] **Step 6: Commit** + +```bash +cd ~/workspaces/plasmodb/ApiCommonModel +git add Model/lib/wdk/ontology/individuals.txt +git commit -m "Categorize VariationsByIsolateGroup under Genetic Variation + +Same placement as VariationBySourceId: parent topic_0199, targetType search, +menu + webservice scopes. Requires wb ontology, not wb model -- a stale OWL +leaves the search uncategorized with no error anywhere." +``` + +--- + +### Task 9: End-to-end verification in the browser + +Everything in the params and queries is already verified by execution against the database. What only a live run can establish is the chain: hidden param → EDA queries → filter tree → plugin → results → record pages. **This is also the first real exercise of the `ApiCommonWebService` plumbing** (`/dnaseq` in the search dir, and the `variation_sample_meta` param name), which that repo's spec could not verify on its own. + +**Files:** none — this task changes nothing. + +- [ ] **Step 1: Mark the logs** + +```bash +cd ~/workspaces/agentic-veupath-dev && bin/veup-logs.sh plasmodb mark vbig +``` + +- [ ] **Step 2: Open the site and confirm which site you are on** + +Load `https://jbrestel.plasmodb.org/a/app` in Chrome and authenticate past the pre-release gate if prompted. Then, before trusting anything: + +```javascript +window.location.origin +``` + +Expected: `"https://jbrestel.plasmodb.org"`. If it reads `https://veupathdb.org`, **stop** — the tab bounced to autologin and every relative fetch from here answers for production. Authenticate and reload. + +- [ ] **Step 3: Confirm the search is registered for this site** + +From that authenticated page: + +```javascript +fetch('/a/service/record-types/variation') + .then(r => r.json()) + .then(d => d.searches.map(s => s.fullName).filter(n => n.includes('Variation'))) +``` + +Expected: includes `VariationQuestions.VariationsByIsolateGroup`. `/record-types` is project-filtered, so this — not the category tree — is the source of truth for whether this site has the search. + +If that 404s, the record type's URL segment is not `variation`; get the real one from `fetch('/a/service/record-types').then(r=>r.json()).then(d=>d.map(t=>t.urlSegment))` and retry. + +- [ ] **Step 4: Reach the search page and record its real URL** + +Navigate through the site's Searches menu (Genetic Variation → Differences Within a Group of Samples) rather than guessing a URL. **Record the URL you land on** in your report; later tasks and the other three searches will want it. + +- [ ] **Step 5: The organism param** + +Expected: a treeBox offering `Plasmodium falciparum 3D7` as the only selectable leaf, under branch nodes. If the tree is empty, the vocabulary query returned nothing for this project — re-run Task 2 Step 1. + +- [ ] **Step 6: The samples filter — the first proof the hidden param works** + +Select `Plasmodium falciparum 3D7`. Expected: the Samples filter populates with **216 samples** under a **7-category** tree (Provenance and identity, Organism under investigation, Specimen and culture, Collection event, Host, Collection location, Alignment statistics), with 20 leaf variables among them. + +This step exercises the hidden suffix param and both EDA queries end to end. Failure modes and what they mean: + +| symptom | cause | +|---|---| +| filter empty, no error | the suffix vocabulary returned zero rows — Task 3 Step 1 | +| `relation "eda.attributevalue_..." does not exist` | the suffix is wrong or the table is absent — Task 3 Step 2 | +| `Parent ontology ID 'sample' ... cannot be found` | the `CASE` on the parent is missing or wrong — Task 4 Step 2 | +| `The following ontology items have no children ... null item type` | the type mapping dropped a leaf's type — Task 4 Step 2 | + +- [ ] **Step 7: Run the search — the first exercise of the Java plumbing** + +Select at least two samples (accepting all 216 is fine), leave the thresholds at their defaults (80% RFT, minor allele frequency 0, percent called 20), and submit. + +Expected: a result page with rows, showing the `PercentMinorAlleles` and `PercentIsolateCalls` columns. + +| symptom | cause | +|---|---| +| `Organism dir does not exist` | the HSSS path — `buildNumber` (Task 1) or the plumbing's `/dnaseq` search dir | +| missing required parameter | the filterParam name does not match `getStrainFilterParamName()` (Task 4/6 Step 1) | +| `expected 4 columns` | the plugin's results-file contract — a plumbing problem, not a model one | + +Selecting exactly two samples is also worth one run: with `minSelectedCount="2"`, one sample must be rejected client-side. + +- [ ] **Step 8: Confirm the IDs resolve** + +Expected: IDs of the form `Variant__` (e.g. `Variant_Pf3D7_01_v3_100057`). Click one through to its record page and confirm it renders. This is the proof that the plumbing spec's ID-construction work holds through a real search — the plugin's `idPrefix` and `hsssReconstructSnpId` join must agree with what `VariationAttributes.source_id` actually contains. + +- [ ] **Step 9: Read the logs** + +```bash +cd ~/workspaces/agentic-veupath-dev && bin/veup-logs.sh plasmodb since vbig --quiet +``` + +Expected: the error logs report `silent:`. A healthy page load leaves them silent, so anything there is worth reading even if the page looked fine. + +- [ ] **Step 10: Report** + +Report to John: the search page URL, the sample and category counts you actually saw, the row count returned, one example variation ID that resolved to a record page, and the log verdict. If any step failed, report the symptom and the diagnosis from the tables above rather than guessing at a fix. + +--- + +## What this plan does not do + +Named so nobody thinks they were forgotten: + +- **The other three HSSS searches** — `ByLocation`, `ByGeneIds`, `ByTwoIsolateGroups`. Each reuses everything built in Tasks 2–5. +- **`FindMajorAllelesPlugin`'s param rename.** It hardcodes `ngsSnp_strain_meta_a` / `_m` as its own required-param contract. Deferred to the `ByTwoIsolateGroups` spec, which **must not forget it**. +- **Deleting the dead `snpParams.xml`** and the rest of the commented-out snp block. Once Task 5 copies the four params out, it has no remaining reason to exist — but removal has its own blast radius (`recordParams.xml`, `spanQuestions.xml`, `SnpsBySpanLogic`). +- **`snpParams.MinPercentMajorAlleles`, the `*Two` variants, and the wizard params** — used only by the two-group searches, so they migrate with `ByTwoIsolateGroups`. +- **Per-strain / VCF data.** `build-71/.../dnaseq/` also holds `vcf` and `bigwig` — inputs for the deferred strain tables on the variation record. +- **Reviving the HSSS test harnesses.** Both are broken (see the plumbing spec §4); this search's verification is the browser. diff --git a/docs/superpowers/plans/2026-08-05-variations-by-location-and-gene-ids.md b/docs/superpowers/plans/2026-08-05-variations-by-location-and-gene-ids.md new file mode 100644 index 000000000..70a3bfbb9 --- /dev/null +++ b/docs/superpowers/plans/2026-08-05-variations-by-location-and-gene-ids.md @@ -0,0 +1,964 @@ +# `VariationsByLocation` + `VariationsByGeneIds` Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Add the two region-restricted HSSS variation searches — by genomic location and by gene list — reusing the param machinery already built for `VariationsByIsolateGroup`. + +**Architecture:** One constant renamed in `ApiCommonWebService`; in `ApiCommonModel`, one new chromosome param plus its vocabulary query, two `processQuery`s, two questions, two category-ontology rows. Everything else is reused verbatim. The design is `docs/superpowers/specs/2026-08-05-variations-by-location-and-gene-ids-design.md` — read it; this plan implements it and does not restate its reasoning. + +**Tech Stack:** WDK model XML; Java (one string constant); PostgreSQL (`unidb_shu_a`); the `agentic-veupath-dev` control plane for remote builds on `cedar`; Claude in Chrome for verification. + +--- + +## Orientation + +Read this before Task 1 even if you implemented the previous search — two items are new. + +**Repos.** This work spans two, both on branch **`dnaseq-merge-experiments`**, never `main`: + +| | | +|---|---| +| `~/workspaces/plasmodb/ApiCommonWebService` | Task 1 only (one Java constant) | +| `~/workspaces/plasmodb/ApiCommonModel` | Tasks 2–5 | +| `~/workspaces/agentic-veupath-dev` | control plane — run `bin/veup-*.sh` from **here** | + +Local edits reach `cedar` through a running `mutagen` sync. You never copy files. Builds run remotely. + +**There is no unit-test framework for WDK model XML.** Don't look for one. Verification is `psql` for SQL, a one-second `ElementTree` parse for well-formedness, a remote build for reference resolution, and `wdkXml` to prove presence in the *assembled* model rather than in a file. + +```bash +psql -h localhost -p 5432 -d unidb_shu_a # read-only; every query here is a SELECT +``` + +> **Never** `INSERT`/`UPDATE`/`DELETE`/`ALTER` or touch an index in any schema but `jbrestel`. + +**Five traps, all of which have already cost time on this feature:** + +1. **Flags go BEFORE the profile name.** `bin/veup-build.sh plasmodb wb model --dry-run` silently drops the flag **and runs for real**. A real dry run prints `DRYRUN:`-prefixed lines. +2. **XML forbids `--` inside ``.** Every comment below is checked. If you reword one, keep double hyphens out or the file will not parse. Text inside `` is exempt. +3. **`wdkXml` prints attributes single-quoted** (`name='x'`). A double-quoted grep pattern matches nothing regardless of what the model contains. +4. **`dynamicAttributes` is mandatory** on any question whose `attributesList` names a `processQuery`'s `wsColumn`s. Both questions here need one. Omitting it fails the build with `Summary attribute field [PercentMinorAlleles] ... is invalid`. +5. **A remote grep for a `$`-containing pattern gets expanded by the remote shell** unless single-quoted on the remote side. Use `ssh host "... '\$foo' ..."`. + +**Do not "fix" the `No Match` guard.** `FindPolymorphismsWithSeqFilterPlugin` contains `if (seq.contains("No Match")) seq = chromosome;`, which looks like dead code. It is live: `sharedParams.sequenceId` has twelve per-project `` children each carrying `allowEmpty="true" emptyValue="No Match"`, and WDK substitutes that literal for an empty box. Design §4.3. Leave it exactly as it is. + +--- + +## File Structure + +| File | Change | Responsibility | +|---|---|---| +| `ApiCommonWebService/.../FindPolymorphismsWithSeqFilterPlugin.java:17` | Modify | the chromosome param name contract (Task 1) | +| `ApiCommonModel/Model/lib/wdk/model/questions/params/variationParams.xml` | Add 1 query to `VariationVQ`, 1 param to `variationParams` | the chromosome param (Task 2) | +| `ApiCommonModel/Model/lib/wdk/model/questions/queries/variationQueries.xml` | Add 2 `processQuery`s | plugin bindings (Tasks 3, 4) | +| `ApiCommonModel/Model/lib/wdk/model/questions/variationQuestions.xml` | Add 2 questions | the user-facing searches (Tasks 3, 4) | +| `ApiCommonModel/Model/lib/wdk/ontology/individuals.txt` | Append 2 rows | category placement (Task 5) | + +--- + +### Task 1: Rename the chromosome param constant in `ApiCommonWebService` + +**Files:** +- Modify: `WSFPlugin/src/main/java/org/apidb/apicomplexa/wsfplugin/highspeedsnpsearch/FindPolymorphismsWithSeqFilterPlugin.java:17` + +- [ ] **Step 1: Confirm you are changing exactly one of the two `PARAM_CHROMOSOME` declarations** + +```bash +cd ~/workspaces/plasmodb/ApiCommonWebService +git branch --show-current +grep -rn "PARAM_CHROMOSOME =" WSFPlugin/src/main/java/org/apidb/apicomplexa/wsfplugin/highspeedsnpsearch/ +``` + +Expected: branch `dnaseq-merge-experiments`, and exactly two declarations: + +``` +FindChipPolymorphismsWithSeqFilterPlugin.java:20: ... PARAM_CHROMOSOME = "chromosomeOptional"; +FindPolymorphismsWithSeqFilterPlugin.java:17: ... PARAM_CHROMOSOME = "chromosomeOptionalForNgsSnps"; +``` + +**Only the second changes.** The chip plugin uses `chromosomeOptional`, a different param serving live chip-snp searches; touching it would break them. + +- [ ] **Step 2: Make the change** + +`FindPolymorphismsWithSeqFilterPlugin.java` line 17: + +```java + public static final String PARAM_CHROMOSOME = "chromosomeOptionalForVariations"; +``` + +Change nothing else in the file. In particular leave line 44's +`if (seq.contains("No Match")) seq = chromosome;` untouched — see Orientation. + +- [ ] **Step 3: Verify the diff is one line** + +```bash +cd ~/workspaces/plasmodb/ApiCommonWebService && git diff --stat && git diff +``` + +Expected: `1 file changed, 1 insertion(+), 1 deletion(-)`, showing only the string literal change. + +- [ ] **Step 4: Commit** + +```bash +cd ~/workspaces/plasmodb/ApiCommonWebService +git add WSFPlugin/src/main/java/org/apidb/apicomplexa/wsfplugin/highspeedsnpsearch/FindPolymorphismsWithSeqFilterPlugin.java +git commit -m "Rename the chromosome param contract for variation searches + +FindPolymorphismsWithSeqFilterPlugin serves VariationsByLocation now. Its +sibling contract, the strain filter, was already renamed to +variation_sample_meta; leaving this one as chromosomeOptionalForNgsSnps would +give a single processQuery two differently-named eras of the same plugin and +invite the next reader to re-derive that the snp name is meaningless. + +The chip plugin's own PARAM_CHROMOSOME (chromosomeOptional) is untouched: it +serves live chip-snp searches." +``` + +- [ ] **Step 5: Build and install** + +```bash +cd ~/workspaces/agentic-veupath-dev && \ + ssh -o LogLevel=ERROR "$(python3 bin/resolve.py --profile profiles/plasmodb.yml --field host)" \ + "bash -lc 'source /var/www/jbrestel.plasmodb.org/etc/setenv && bld ApiCommonWebService'" +``` + +Expected: `BUILD SUCCESSFUL` (roughly 1–2 minutes). `Test-Installation` is not in the default +depends list, so the non-compiling JUnit module is not built. + +- [ ] **Step 6: Reload, and confirm the new name reached the installed jar** + +WSF plugins are loaded by the webapp; the constant does not take effect until a reload. + +```bash +cd ~/workspaces/agentic-veupath-dev && bin/veup-build.sh plasmodb reload +``` + +Expected: `OK - Reloaded application at context path [/plasmo.jbrestel]`. + +Jar entries are compressed, so **`grep` over the lib directory finds nothing whether or not the +string is there** — a check that can only fail. Read the class out of the jar instead, and test +for both strings; the old one being *gone* is the stronger signal: + +```bash +ssh cedar "bash -lc 'J=/var/www/PlasmoDB/plasmo.jbrestel/webapp/WEB-INF/lib/api-common-websvc-wsfplugin-1.0.0.jar; \ + C=org/apidb/apicomplexa/wsfplugin/highspeedsnpsearch/FindPolymorphismsWithSeqFilterPlugin.class; \ + echo -n \"new=\"; unzip -p \$J \$C | strings | grep -c chromosomeOptionalForVariations; \ + echo -n \"old=\"; unzip -p \$J \$C | strings | grep -c chromosomeOptionalForNgsSnps'" +``` + +Expected: `new=1` and `old=0`. Also check the jar's mtime matches the build you just ran +(`ls -l $J`). If the jar name has changed, find it with +`ssh cedar 'ls /var/www/PlasmoDB/plasmo.jbrestel/webapp/WEB-INF/lib/ | grep wsfplugin'`. + +**A pass here is what makes Task 6 meaningful** — with the old string installed, the search +fails at run time with a missing-required-parameter error and the model looks wrong when it is +not. + +--- + +### Task 2: The chromosome param and its vocabulary + +**Files:** +- Modify: `ApiCommonModel/Model/lib/wdk/model/questions/params/variationParams.xml` + +- [ ] **Step 1: Run the vocabulary SQL — this is the test** + +The model substitutes the organism param at run time; run it with the value inlined: + +```bash +psql -h localhost -p 5432 -d unidb_shu_a -c " +SELECT * FROM ( + SELECT DISTINCT s.chromosome AS term, + s.source_id AS internal, + s.chromosome_order_num + FROM webready.GenomicSeqAttributes_p s + WHERE s.organism = 'Plasmodium falciparum 3D7' + AND s.chromosome IS NOT NULL + UNION + SELECT 'Choose chromosome' AS term, 'choose' AS internal, -1 AS chromosome_order_num +) t +ORDER BY chromosome_order_num" +``` + +Expected: **15 rows** — the sentinel first (`chromosome_order_num = -1`), then `01` through `14` +with internals `Pf3D7_01_v3` … `Pf3D7_14_v3`. + +The `internal` values are the load-bearing part: `Pf3D7_01_v3` is exactly the sequence +identifier the HSSS files carry (the live `ByIsolateGroup` run returned +`Variant_Pf3D7_01_v3_1`), so the value feeds the position filter with no mapping. + +- [ ] **Step 2: Confirm why the predicate is `organism` and not `org_abbrev`** + +```bash +psql -h localhost -p 5432 -d unidb_shu_a -c " +SELECT count(*) FROM webready.GenomicSeqAttributes_p +WHERE org_abbrev = 'Plasmodium falciparum 3D7'" +``` + +Expected: `0`. The snp original keyed on `org_abbrev`, but our organism param carries the +**taxon name** because the HSSS plugin resolves it through `sres.TaxonName` to build the +webservices path. That zero is why the column changed. + +- [ ] **Step 3: Add the vocabulary query** + +In `variationParams.xml`, inside the existing `VariationVQ` querySet, after +`SampleOntologyByStudy`: + +```xml + + + + + + + + + +``` + +`noTranslation="true"` passes the organism param's **term** rather than its internal, matching +`EdaSampleTableSuffix` and the snp-era precedent. The SQL supplies the quotes. + +Note the query declares only `internal` and `term` as columns even though the SQL selects +`chromosome_order_num` — that third column exists solely to drive `ORDER BY`, and the snp +original did the same. + +- [ ] **Step 4: Add the param** + +Inside the `variationParams` paramSet, after `MinPercentIsolateCalls`: + +```xml + + + + + + + + + +``` + +- [ ] **Step 5: Verify the XML parses and the pieces landed in the right containers** + +```bash +cd ~/workspaces/plasmodb/ApiCommonModel && python3 - <<'PY' +import xml.etree.ElementTree as T +r = T.parse('Model/lib/wdk/model/questions/params/variationParams.xml').getroot() +for ps in r.findall('paramSet'): + print('paramSet', ps.get('name'), [(c.tag, c.get('name')) for c in ps]) +for qs in r.findall('querySet'): + print('querySet', qs.get('name'), [q.get('name') for q in qs.findall('sqlQuery')]) +PY +``` + +Expected: `variationParams` now lists `variation_id`, `eda_sample_table_suffix`, +`variation_sample_meta`, `WebServicesPath`, `ReadFrequencyPercent`, `MinPercentMinorAlleles`, +`MinPercentIsolateCalls`, `chromosomeOptionalForVariations`; and `VariationVQ` lists +`EdaSampleTableSuffix`, `SamplesMetadataByStudy`, `SampleOntologyByStudy`, +`ChromosomeForVariations`. + +- [ ] **Step 6: Commit** + +```bash +cd ~/workspaces/plasmodb/ApiCommonModel +git add Model/lib/wdk/model/questions/params/variationParams.xml +git commit -m "Add chromosomeOptionalForVariations param and its vocabulary + +Rewritten, not reused. The snp original's vocabulary reaches +organismVQ.withNgsSNPs, which reads the nonexistent apidbtuning.snpstrains, +and keys on org_abbrev while our organism param carries the taxon name (0 rows +if you try it). Keyed on organism instead: 15 rows in 26ms. + +internal is the sequence source_id (chromosome 01 gives Pf3D7_01_v3), which is +the identifier the HSSS files use, so the value feeds the position filter with +no mapping layer." +``` + +--- + +### Task 3: `VariationsByLocation` — query, question, build + +**Files:** +- Modify: `ApiCommonModel/Model/lib/wdk/model/questions/queries/variationQueries.xml` +- Modify: `ApiCommonModel/Model/lib/wdk/model/questions/variationQuestions.xml` + +- [ ] **Step 1: Confirm the plugin's required params before wiring to them** + +```bash +cd ~/workspaces/plasmodb/ApiCommonWebService +sed -n '15,35p' WSFPlugin/src/main/java/org/apidb/apicomplexa/wsfplugin/highspeedsnpsearch/FindPolymorphismsWithSeqFilterPlugin.java +``` + +Expected: `PARAM_CHROMOSOME = "chromosomeOptionalForVariations"` (Task 1's change), and +`getExtraParamNames()` returning all four of chromosome, sequence, start point, end point. +All four are **required** — none may be omitted from the `processQuery`. + +- [ ] **Step 2: Add the process query** + +In `variationQueries.xml`, inside the `VariationsBy` querySet, after +`VariationsByIsolateGroup`: + +```xml + + + + + + + + The Organism defines the species identity of the samples and the genome + against which each sample's variants were called. + + + + + + + + + + + + + + + + + + + +``` + +- [ ] **Step 3: Add the question** + +In `variationQuestions.xml`, inside the `VariationQuestions` questionSet, after +`VariationsByIsolateGroup`: + +```xml + + + + + + + + + + + + +
    + + Each sample's sequencing reads are aligned to the reference genome (Organism) + and variants are recorded for each sample based on the Read Frequency + Threshold. Then, scanning variant locations within your region across the + group of samples, variants are returned if the Minor Allele Frequency and the + Percent samples with a base call are met. + +

    Defining the region: Either choose a Chromosome, or enter a Genomic + sequence ID. A sequence ID you enter takes precedence; the Chromosome menu is + used when you leave the sequence box empty. Start and End restrict the region + further, and an End of 0 means "to the end of the sequence".

    + +

    Organism: The Organism parameter defines the species of the samples + and the genome in which the variants are determined. Choosing an Organism + focuses the Samples parameter to the samples of that organism.

    + +

    Samples: By default the group includes all samples from the Organism + you chose; you may narrow it using the sample characteristics. At least two + samples are required, since polymorphism within a group of one is undefined.

    + +

    Read frequency threshold: An allele is called for a sample at a + location if that fraction of the sample's aligned reads support it. This + matters most for diploid or aneuploid organisms, where heterozygous positions + are expected near 50%.

    + +

    Minor allele frequency: Among the qualifying calls at a location, + the minor allele frequency is the percent that are not the major allele. Use 0 + to find every variant location within the group.

    + +

    Percent samples with a base call: A location is only considered if + this fraction of your selected samples have a qualifying call there.

    + ]]> +     + + + + + + Display the histogram of the values of this attribute + int + + + + + Display the histogram of the values of this attribute + int + + + + + +     +``` + +- [ ] **Step 4: Verify both files parse** + +```bash +cd ~/workspaces/plasmodb/ApiCommonModel && for f in \ + Model/lib/wdk/model/questions/queries/variationQueries.xml \ + Model/lib/wdk/model/questions/variationQuestions.xml; do + python3 -c "import xml.etree.ElementTree as T,sys; T.parse('$f'); print('$f parses')" +done +``` + +Expected: both print `parses`. + +- [ ] **Step 5: Build and confirm the search is in the assembled model** + +```bash +cd ~/workspaces/agentic-veupath-dev && bin/veup-build.sh plasmodb wb model +``` + +Expected: completes with no `WdkModelException`. + +```bash +ssh cedar 'bash -lc "source /var/www/jbrestel.plasmodb.org/etc/setenv && wdkXml -model PlasmoDB"' \ + | grep -E "VariationsBy.VariationsByLocation:" +``` + +Expected: one line listing the query's params and columns — `organismSinglePick`, +`eda_sample_table_suffix`, `chromosomeOptionalForVariations`, `sequenceId`, `start_point`, +`end_point`, `variation_sample_meta`, `WebServicesPath`, `ReadFrequencyPercent`, +`MinPercentMinorAlleles`, `MinPercentIsolateCalls`, and columns `source_id, project_id, +PercentMinorAlleles, PercentIsolateCalls, Phenotype`. + +- [ ] **Step 6: Commit** + +```bash +cd ~/workspaces/plasmodb/ApiCommonModel +git add Model/lib/wdk/model/questions/queries/variationQueries.xml \ + Model/lib/wdk/model/questions/variationQuestions.xml +git commit -m "Add the VariationsByLocation search + +Reuses every param from VariationsByIsolateGroup and adds the region +restriction: chromosome (fallback), sequenceId (takes precedence), start and +end. All four are required by FindPolymorphismsWithSeqFilterPlugin. + +The chromosome fallback works through sequenceId's emptyValue='No Match', +which the plugin tests for by literal; documented in the query comment because +it reads as dead code from either side alone." +``` + +--- + +### Task 4: `VariationsByGeneIds` — query, question, build + +**Files:** +- Modify: `ApiCommonModel/Model/lib/wdk/model/questions/queries/variationQueries.xml` +- Modify: `ApiCommonModel/Model/lib/wdk/model/questions/variationQuestions.xml` + +- [ ] **Step 1: Confirm the plugin's gene-resolution SQL still works against this build** + +The plugin resolves your gene list to genomic intervals itself. Check the table it uses, and +record the interval you will assert against in Task 6: + +```bash +psql -h localhost -p 5432 -d unidb_shu_a -c " +SELECT source_id, sequence_id, start_min, end_max +FROM webready.GeneAttributes_p WHERE source_id = 'PF3D7_1133400'" +``` + +Expected: `PF3D7_1133400 | Pf3D7_11_v3 | 1292966 | 1296696`. If `webready.geneattributes_p` +did not exist, this search could not work at all and the plan would need revisiting. + +- [ ] **Step 2: Add the process query** + +In `variationQueries.xml`, inside the `VariationsBy` querySet, after `VariationsByLocation`: + +```xml + + + + + + + + The Organism defines the species identity of the samples and the genome + against which each sample's variants were called. + + + + + + + + + + + + + + + + + + +``` + +- [ ] **Step 3: Add the question** + +In `variationQuestions.xml`, after `VariationsByLocation`: + +```xml + + + + + + + + + + + + +
    + + Each sample's sequencing reads are aligned to the reference genome (Organism) + and variants are recorded for each sample based on the Read Frequency + Threshold. Then, scanning variant locations within your genes across the group + of samples, variants are returned if the Minor Allele Frequency and the Percent + samples with a base call are met. + +

    Genes: Your gene IDs are resolved to each gene's genomic span, and + variants are returned by position within those spans. A variant in an intron or + UTR of one of your genes is therefore returned, since the span covers the whole + gene rather than only its coding sequence.

    + +

    Organism: The Organism parameter defines the species of the samples + and the genome in which the variants are determined. Choose the organism your + genes belong to.

    + +

    Samples: By default the group includes all samples from the Organism + you chose; you may narrow it using the sample characteristics. At least two + samples are required, since polymorphism within a group of one is undefined.

    + +

    Read frequency threshold: An allele is called for a sample at a + location if that fraction of the sample's aligned reads support it. This + matters most for diploid or aneuploid organisms, where heterozygous positions + are expected near 50%.

    + +

    Minor allele frequency: Among the qualifying calls at a location, the + minor allele frequency is the percent that are not the major allele. Use 0 to + find every variant location within the group.

    + +

    Percent samples with a base call: A location is only considered if + this fraction of your selected samples have a qualifying call there.

    + ]]> +     + + + + + + Display the histogram of the values of this attribute + int + + + + + Display the histogram of the values of this attribute + int + + + + + +     +``` + +- [ ] **Step 4: Verify both files parse** + +```bash +cd ~/workspaces/plasmodb/ApiCommonModel && for f in \ + Model/lib/wdk/model/questions/queries/variationQueries.xml \ + Model/lib/wdk/model/questions/variationQuestions.xml; do + python3 -c "import xml.etree.ElementTree as T,sys; T.parse('$f'); print('$f parses')" +done +``` + +Expected: both print `parses`. + +- [ ] **Step 5: Build and confirm all four searches are in the assembled model** + +```bash +cd ~/workspaces/agentic-veupath-dev && bin/veup-build.sh plasmodb wb model +``` + +Expected: completes with no `WdkModelException`. + +```bash +ssh cedar 'bash -lc "source /var/www/jbrestel.plasmodb.org/etc/setenv && wdkXml -model PlasmoDB"' \ + | grep -oE "VariationsBy\.[A-Za-z]+:" | sort -u +``` + +Expected exactly four: `VariationsBy.VariationBySourceId:`, +`VariationsBy.VariationsByGeneIds:`, `VariationsBy.VariationsByIsolateGroup:`, +`VariationsBy.VariationsByLocation:`. + +- [ ] **Step 6: Commit** + +```bash +cd ~/workspaces/plasmodb/ApiCommonModel +git add Model/lib/wdk/model/questions/queries/variationQueries.xml \ + Model/lib/wdk/model/questions/variationQuestions.xml +git commit -m "Add the VariationsByGeneIds search + +No gene-to-variation join in the model: FindSnpsByGeneIdsPlugin resolves the +gene list to genomic intervals through webready.GeneAttributes_p and HSSS +filters variant positions by interval, so the relationship is positional and +apidb.VariationTranscriptProduct plays no part. + +Per-project gene defaults only for the three projects with variation data +loaded; the rest owe one when their data lands." +``` + +--- + +### Task 5: Category ontology rows + +**Files:** +- Modify: `ApiCommonModel/Model/lib/wdk/ontology/individuals.txt` + +The file is tab-delimited with load-bearing empty fields and a trailing tab. Derive each row by +substitution from the `VariationBySourceId` row; do not type them. + +- [ ] **Step 1: Look at the row you are copying** + +```bash +cd ~/workspaces/plasmodb/ApiCommonModel +grep -n "VariationQuestions.VariationBySourceId" Model/lib/wdk/ontology/individuals.txt | cat -A +``` + +Expected: one match showing `^I` between fields, `topic_0199` as parent, `search` as target +type, `menu` and `webservice` at the end, and a trailing `^I` before `$`. + +- [ ] **Step 2: Append both rows by substitution** + +```bash +cd ~/workspaces/plasmodb/ApiCommonModel +for s in VariationsByLocation VariationsByGeneIds; do + grep "VariationQuestions.VariationBySourceId" Model/lib/wdk/ontology/individuals.txt \ + | sed "s/VariationBySourceId/$s/g" >> Model/lib/wdk/ontology/individuals.txt +done +``` + +- [ ] **Step 3: Verify field counts match the source row** + +```bash +cd ~/workspaces/plasmodb/ApiCommonModel +awk -F'\t' '/VariationQuestions.Variation/ {print NF" fields "$6}' Model/lib/wdk/ontology/individuals.txt +``` + +Expected: four lines, all with the **same** field count, naming +`VariationQuestions.VariationBySourceId`, `...VariationsByIsolateGroup`, +`...VariationsByLocation`, `...VariationsByGeneIds`. A differing count means a shifted column +and a silently misfiled search — stop rather than patching by hand. + +- [ ] **Step 4: Build with `wb ontology`, NOT `wb model`** + +```bash +cd ~/workspaces/agentic-veupath-dev && bin/veup-build.sh plasmodb wb ontology +``` + +Expected: completes, with a line reporting `categories_merged.owl` saved. `wb model` here would +leave the OWL stale and both searches uncategorized, **with no error anywhere**. + +- [ ] **Step 5: Prove the OWL contains both, under the right parent** + +```bash +ssh cedar 'for s in VariationsByLocation VariationsByGeneIds; do echo -n "$s "; \ + grep -A3 "individuals.owl#VariationRecordClasses.VariationRecordClass.VariationQuestions.$s\"" \ + /var/www/jbrestel.plasmodb.org/gus_home/lib/wdk/ontology/categories_merged.owl \ + | grep -c "topic_0199"; done' +``` + +Expected: each prints `1` — the class exists and is `subClassOf topic_0199`. + +- [ ] **Step 6: Commit** + +```bash +cd ~/workspaces/plasmodb/ApiCommonModel +git add Model/lib/wdk/ontology/individuals.txt +git commit -m "Categorize VariationsByLocation and VariationsByGeneIds + +Same placement as the two existing variation searches: parent topic_0199, +targetType search, menu + webservice scopes. No searchCategory on either +question: it appears zero times in the assembled model, so menu grouping comes +from this file alone." +``` + +--- + +### Task 6: Browser verification + +Everything in Task 2 is verified by execution against the database. What only a live run can +establish is the two chains no test covers: the `emptyValue` to chromosome fallback, and gene +IDs through interval filtering to HSSS. + +**Files:** none. + +- [ ] **Step 1: Mark the logs** + +```bash +cd ~/workspaces/agentic-veupath-dev && bin/veup-logs.sh plasmodb mark loc-gene +``` + +- [ ] **Step 2: Open the app and confirm which site you are on** + +Load `https://jbrestel.plasmodb.org/a/app` in Chrome. It **redirects** to the webapp context. +Then, before trusting anything: + +```javascript +({origin: window.location.origin, base: window.location.pathname.split('/')[1]}) +``` + +Expected: `origin` is `https://jbrestel.plasmodb.org` and `base` is `plasmo.jbrestel`. If +origin reads `https://veupathdb.org`, **stop** — the tab bounced to autologin and every +relative fetch from here answers for production. Build all paths below from the `base` you +actually got, not from `/a/`. + +Note: `computer:screenshot` fails on this instance with `Script injection timed out`, including +on known-good pages. Use `javascript_tool` and the service endpoints; do not spend attempts on +screenshots. + +- [ ] **Step 3: Confirm both searches are registered** + +```javascript +const b = window.location.pathname.split('/')[1]; +fetch(`/${b}/service/record-types/variation`).then(r=>r.json()) + .then(d=>d.searches.map(s=>s.fullName).filter(n=>n.startsWith('VariationQuestions'))) +``` + +Expected: all four — `VariationBySourceId`, `VariationsByIsolateGroup`, `VariationsByLocation`, +`VariationsByGeneIds`. `/record-types` is project-filtered, so this is the source of truth. + +- [ ] **Step 4: `VariationsByLocation` — the chromosome vocabulary** + +```javascript +const b = window.location.pathname.split('/')[1]; +const q = await fetch(`/${b}/service/record-types/variation/searches/VariationsByLocation?expandParams=true`).then(r=>r.json()); +const chr = q.searchData.parameters.find(p=>p.name==='chromosomeOptionalForVariations'); +JSON.stringify({count: chr.vocabulary.length, terms: chr.vocabulary.map(v=>v[0])}) +``` + +Expected: 15 entries — `Choose chromosome` plus `01` through `14`. + +- [ ] **Step 5: `VariationsByLocation` — run it with the sequence box EMPTY** + +This is the step that exercises the `emptyValue` to `No Match` to chromosome-fallback chain. +Submit with `chromosomeOptionalForVariations` = the internal for chromosome 01 +(`Pf3D7_01_v3`), `sequenceId` left empty, `start_point` `0`, `end_point` `0`, all samples, +thresholds at 80% / 0 / 20. + +Expected: rows returned, and **every** returned ID begins `Variant_Pf3D7_01_v3_`. Check the +first page of IDs explicitly — a mixture of sequences would mean the region filter was ignored. + +| symptom | cause | +|---|---| +| zero results | the fallback did not fire; check `sequenceId`'s `emptyValue` reached the plugin | +| IDs from other chromosomes | the sequence argument never reached the bash script | +| missing required parameter | Task 1's rename is not installed — re-run Task 1 Steps 5 and 6 | + +- [ ] **Step 6: `VariationsByLocation` — run it with an explicit sequence and window** + +Submit `sequenceId` = `Pf3D7_11_v3`, `start_point` `1292966`, `end_point` `1296696`. + +Expected: fewer rows than Step 5, all with IDs of the form `Variant_Pf3D7_11_v3_` where +`1292966 <= n <= 1296696`. Extract `n` from the IDs and check the min and max against the +bounds rather than eyeballing. + +- [ ] **Step 7: `VariationsByGeneIds` — the first real exercise of `hsssGenomicLocationsFilter`** + +Submit with the PlasmoDB default gene `PF3D7_1133400`, all samples, thresholds as above. + +Expected: a non-empty result set, every ID of the form `Variant_Pf3D7_11_v3_` with +`1292966 <= n <= 1296696` — the same window as Step 6, because that is this gene's span. + +**A wrong ID form here surfaces as zero results with no error**, which is exactly why a +non-empty set is the assertion. The installed `hsssGenomicLocationsFilter` was confirmed to +carry the underscore ID join; if this returns nothing, re-check it with the pattern +single-quoted on the remote side: + +```bash +ssh cedar "grep -cF '\${contigSourceId}_\${location}' /var/www/jbrestel.plasmodb.org/gus_home/bin/hsssGenomicLocationsFilter" +``` + +Expected: `2`. (Without the escaping, the remote shell expands the variables and you count `_`.) + +- [ ] **Step 8: Confirm an ID resolves to a record page** + +Navigate to `//app/record/variation/` and confirm it renders +with its Genomic Location / Genetic variation / DNA polymorphism sections and no error. + +- [ ] **Step 9: Confirm both searches are in the category tree** + +```javascript +const b = window.location.pathname.split('/')[1]; +const c = await fetch(`/${b}/service/ontologies/Categories`).then(r=>r.json()); +const out=[]; +(function walk(n,parent){const p=n.properties||{};const nm=(p.name||[])[0]; + if(nm && nm.startsWith('VariationQuestions')) out.push({name:nm, parent}); + (n.children||[]).forEach(ch=>walk(ch,(p['EuPathDB alternative term']||p.label||[])[0]||parent)); +})(c.tree,'ROOT'); +JSON.stringify(out,null,1) +``` + +Expected: all four searches, each with `parent` `"Genetic variation"`. + +- [ ] **Step 10: Read the logs** + +```bash +cd ~/workspaces/agentic-veupath-dev && bin/veup-logs.sh plasmodb since loc-gene --quiet +``` + +Expected: the error logs report `silent:`. Also confirm the plugin used the paths you expect: + +```bash +cd ~/workspaces/agentic-veupath-dev && bin/veup-logs.sh plasmodb since loc-gene 2>&1 \ + | grep -oE "dnaseq/readFreq[0-9]+|hsssGenerate[A-Za-z]+" | sort | uniq -c +``` + +Expected: `hsssGeneratePolymorphismScript` for the two by-location runs and +`hsssGenerateGenomicLocationsScript` for the by-gene run. + +**Any ERROR lines you cause yourself by hand-rolling malformed service requests must be +reported as such, not as "the logs were silent."** That happened on the previous search. + +- [ ] **Step 11: Report** + +Report: the four registered searches; the chromosome vocabulary count; for each of the three +runs, the row count and the observed min/max location against the expected window; one ID that +resolved to a record page; the category-tree parents; the log verdict, distinguishing +self-inflicted errors from real ones. If a step failed, give the symptom and the diagnosis from +the tables above rather than guessing at a fix. + +--- + +## Out of scope + +- **`VariationsByTwoIsolateGroups`** — its own spec and plan. Different plugin + (`FindMajorAllelesPlugin`), 11 `wsColumn`s, five doubled threshold params, and **two new + EDA-driven filter params**, because `sharedParams.ngsSnp_strain_meta_a`/`_m` sit inside a + commented-out region and are not in the model. That spec must not forget the plugin's + hardcoded `_a`/`_m` names. +- **`NgsSnpsByTwoIsolateGroupsWiz`** — a sixth search using the `*_wiz` params; decide whether + to port it at all alongside `ByTwoIsolateGroups`. +- **Per-gene coding consequences** from `apidb.VariationTranscriptProduct`. A feature, not a port. +- **Deleting the dead `snpParams.xml`** and the commented-out snp regions of `sharedParams.xml`. +- **`ReadFrequencyPercent` as a functional parameter.** On haploid organisms all four + `readFreq*` directories hold identical data, because the upstream caller runs + `freebayes --min-alternate-fraction 0.8`. The param selects its directory correctly. Do not + attempt to verify that changing it changes results; it will not, and that is not a bug in + these searches. Design §9. diff --git a/docs/superpowers/plans/2026-08-05-variations-by-two-isolate-groups.md b/docs/superpowers/plans/2026-08-05-variations-by-two-isolate-groups.md new file mode 100644 index 000000000..28fa3fc56 --- /dev/null +++ b/docs/superpowers/plans/2026-08-05-variations-by-two-isolate-groups.md @@ -0,0 +1,810 @@ +# `VariationsByTwoIsolateGroups` Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Add the fifth and last ported snp search — find loci whose major allele differs between two user-chosen groups of samples — completing the `snp` to `variation` migration. + +**Architecture:** Two constants renamed in `ApiCommonWebService`; in `ApiCommonModel`, two new filter params (sharing the existing EDA queries — **no new SQL**), four new threshold params, one `processQuery`, one question, one ontology row. The design is `docs/superpowers/specs/2026-08-05-variations-by-two-isolate-groups-design.md` — read it; this plan implements it and does not restate its reasoning. + +**Tech Stack:** WDK model XML; Java (two string constants); the `agentic-veupath-dev` control plane for remote builds on `cedar`; Claude in Chrome for verification. + +--- + +## Orientation + +**Repos**, both on branch **`dnaseq-merge-experiments`**, never `main`: + +| | | +|---|---| +| `~/workspaces/plasmodb/ApiCommonWebService` | Task 1 only | +| `~/workspaces/plasmodb/ApiCommonModel` | Tasks 2–4 | +| `~/workspaces/agentic-veupath-dev` | control plane — run `bin/veup-*.sh` from **here** | + +Local edits reach `cedar` through a running `mutagen` sync. Builds run remotely. + +**There is no unit-test framework for WDK model XML.** Verification is a structural XML check, a remote build (proves references resolve), `wdkXml` (proves presence in the *assembled* model), and the browser. + +**Six traps, every one of which has already cost time on this feature:** + +1. **Flags go BEFORE the profile name.** `bin/veup-build.sh plasmodb wb model --dry-run` silently drops the flag **and runs for real**. Commands below are complete; add nothing. +2. **XML forbids `--` inside ``.** All comments below are checked; keep double hyphens out if you reword. `` is exempt. +3. **`wdkXml` prints attributes single-quoted** (`name='x'`). A double-quoted grep pattern matches nothing regardless of model content. +4. **`dynamicAttributes` is mandatory** for any `wsColumn` named in `attributesList`. Ten of them here. Omitting the block fails the build with `Summary attribute field [...] is invalid`. +5. **Jar entries are compressed** — `grep` over `WEB-INF/lib` finds nothing whether or not a string is present. Unzip the class (Task 1 Step 6). +6. **A remote grep for a `$`-containing pattern** gets expanded by the remote shell unless single-quoted on the remote side: `ssh host "... '\$foo' ..."`. + +**This search's own trap:** `uniq-value-params` (Task 3 Step 3) forbids Set A = Set B. Its absence **cannot be detected by any build or service check** — only by noticing in the browser that the form accepts A = B. Do not drop it, and do verify it in Task 5. + +--- + +## File Structure + +| File | Change | Responsibility | +|---|---|---| +| `ApiCommonWebService/.../FindMajorAllelesPlugin.java:20,24` | Modify 2 lines | the two filter param name contracts (Task 1) | +| `ApiCommonModel/.../params/variationParams.xml` | Add 2 filterParams + 4 threshold params | the doubled params (Task 2) | +| `ApiCommonModel/.../queries/variationQueries.xml` | Add 1 `processQuery` | plugin binding, 12 `wsColumn`s (Task 3) | +| `ApiCommonModel/.../variationQuestions.xml` | Add 1 question | 13 summary columns, 10 dynamic attributes, `uniq-value-params` (Task 3) | +| `ApiCommonModel/Model/lib/wdk/ontology/individuals.txt` | Append 1 row | category placement (Task 4) | + +No file gains new SQL. Both new filter params reuse `VariationVQ.SamplesMetadataByStudy` and +`SampleOntologyByStudy`. + +--- + +### Task 1: Rename the two filter param constants + +**Files:** +- Modify: `WSFPlugin/src/main/java/org/apidb/apicomplexa/wsfplugin/highspeedsnpsearch/FindMajorAllelesPlugin.java:20` and `:24` + +- [ ] **Step 1: Confirm the two lines and that nothing else references the old names** + +```bash +cd ~/workspaces/plasmodb/ApiCommonWebService +git branch --show-current +grep -rn "ngsSnp_strain_meta" WSFPlugin/src/main/java/ +``` + +Expected: branch `dnaseq-merge-experiments`, and exactly two hits, both in `FindMajorAllelesPlugin.java`: + +``` +:20: public static final String PARAM_STRAIN_FILTER_A = "ngsSnp_strain_meta_a"; +:24: public static final String PARAM_STRAIN_FILTER_B = "ngsSnp_strain_meta_m"; +``` + +If any other Java file references those strings, stop and report — the blast radius would be larger than the design assumed. + +- [ ] **Step 2: Make both changes** + +```java + public static final String PARAM_STRAIN_FILTER_A = "variation_sample_meta_a"; +``` +```java + public static final String PARAM_STRAIN_FILTER_B = "variation_sample_meta_b"; +``` + +Note `_m` becomes `_b`, not `_m`. Change nothing else in the file — in particular leave the +Set B strains handling alone, including the missing null check the design records as +deliberately out of scope. + +- [ ] **Step 3: Verify the diff is two lines and the old names are gone** + +```bash +cd ~/workspaces/plasmodb/ApiCommonWebService && git diff --stat && git diff +grep -rc "ngsSnp_strain_meta" WSFPlugin/src/main/java/ 2>/dev/null | grep -v ':0' || echo "old names gone" +``` + +Expected: `1 file changed, 2 insertions(+), 2 deletions(-)`, and `old names gone`. + +- [ ] **Step 4: Commit** + +```bash +cd ~/workspaces/plasmodb/ApiCommonWebService +git add WSFPlugin/src/main/java/org/apidb/apicomplexa/wsfplugin/highspeedsnpsearch/FindMajorAllelesPlugin.java +git commit -m "Rename the two sample-group param contracts for variation searches + +FindMajorAllelesPlugin serves VariationsByTwoIsolateGroups now. It extends +HighSpeedSnpSearchAbstractPlugin directly rather than FindPolymorphismsPlugin, +so these two names are its own constants and were not covered by the earlier +strain-filter rename. + +The odd _m becomes _b: nothing in the plugin distinguishes it beyond being the +second group, and its prompts already read Set B. With this, no snp-era param +name survives in any variation search." +``` + +- [ ] **Step 5: Build and install** + +```bash +cd ~/workspaces/agentic-veupath-dev && \ + ssh -o LogLevel=ERROR "$(python3 bin/resolve.py --profile profiles/plasmodb.yml --field host)" \ + "bash -lc 'source /var/www/jbrestel.plasmodb.org/etc/setenv && bld ApiCommonWebService'" +``` + +Expected: `BUILD SUCCESSFUL`, 1–2 minutes. + +- [ ] **Step 6: Reload and verify the installed jar** + +```bash +cd ~/workspaces/agentic-veupath-dev && bin/veup-build.sh plasmodb reload +``` + +Expected: `OK - Reloaded application at context path [/plasmo.jbrestel]`. + +```bash +ssh cedar "bash -lc 'J=/var/www/PlasmoDB/plasmo.jbrestel/webapp/WEB-INF/lib/api-common-websvc-wsfplugin-1.0.0.jar; \ + C=org/apidb/apicomplexa/wsfplugin/highspeedsnpsearch/FindMajorAllelesPlugin.class; \ + for s in variation_sample_meta_a variation_sample_meta_b ngsSnp_strain_meta_a ngsSnp_strain_meta_m; do \ + echo -n \"\$s=\"; unzip -p \$J \$C | strings | grep -c \$s; done'" +``` + +Expected: `variation_sample_meta_a=1`, `variation_sample_meta_b=1`, and **both old names `=0`**. +Remember jar entries are compressed, so `grep` over the lib directory would find nothing either +way — read the class out, as here. + +**A pass here is what makes Task 5 meaningful.** With an old string installed, the search fails +as a missing required parameter and the model looks wrong when it is fine. + +--- + +### Task 2: The two filter params and four threshold params + +**Files:** +- Modify: `ApiCommonModel/Model/lib/wdk/model/questions/params/variationParams.xml` + +- [ ] **Step 1: Confirm the EDA queries you are about to share already exist** + +No new SQL is written in this task; both filters point at the queries the one-group search uses. + +```bash +cd ~/workspaces/plasmodb/ApiCommonModel && python3 -c " +import xml.etree.ElementTree as T +r=T.parse('Model/lib/wdk/model/questions/params/variationParams.xml').getroot() +for qs in r.findall('querySet'): + print(qs.get('name'), [q.get('name') for q in qs.findall('sqlQuery')]) +" +``` + +Expected: `VariationVQ ['EdaSampleTableSuffix', 'SamplesMetadataByStudy', 'SampleOntologyByStudy', 'ChromosomeForVariations']`. + +- [ ] **Step 2: Add the two filter params** + +Inside the `variationParams` paramSet, after `chromosomeOptionalForVariations`: + +```xml + + + + + + + Select the first group of samples to compare. Use the sample characteristics + to narrow the group, or accept all samples for the organism you chose. + + + + + + Select the second group of samples to compare. It must differ from Set A; + comparing a group against itself returns nothing useful. + + +``` + +- [ ] **Step 3: Add the four threshold params** + +After the two filter params: + +```xml + + + + + + + + This parameter applies to the Set A aligned sample sequences. When a Set A + locus has a major allele frequency greater than or equal to this value, it + will be compared to the equivalent locus in Set B samples. Note that 100% is + permissible and is the most stringent setting, since the search first + identifies an allele in this set and then compares it with the allele in + Set B. See the Description below the Get Answer button for more. + + + \d\d?|100 + + + + + This parameter applies to the aligned sample sequences of Set B. When a Set B + locus has a major allele frequency greater than or equal to this value, it + will be compared to the equivalent locus in Set A samples. Note that 100% is + permissible, since the search first identifies loci from Set A and then + compares them with loci from Set B. See the Description below the Get Answer + button for more. + + + \d\d?|100 + + + + + This parameter applies to the Set B aligned sample sequences. At any given + nucleotide position, some samples in Set B may not have data supporting a + call because the Read Frequency Threshold was not met. This defines the + fraction of Set B samples that must have a base call before a locus is + returned for that position, based on the remaining samples that do have data. + See the Description below for more information. + + + \d\d?|100 + + + + + + This parameter applies to the sequencing reads of individual samples in Set B + and defines a stringency for data supporting a variant call between a sample + and the reference genome (Organism). Each nucleotide position of each sample + is compared to the reference genome and a call is made if the portion of the + sample's aligned reads that support the variant is above the Read Frequency + Threshold (RFT). Find high quality haploid variants with 80% RFT or + heterozygous diploid/aneuploid variants with 40%. See the Description below + for more. + + + + 80% + 80 + + + 60% + 60 + + + 40% + 40 + + + 20% + 20 + + + +``` + +Note `MinPercentIsolateCallsTwo`'s prompt has been harmonised with `MinPercentIsolateCalls`' +wording ("Percent samples with a base call >= ") rather than kept as the original's terser "Min +percent isolates with calls >= ". The design flagged this as the implementer's call; the two +prompts sit side by side in one form, so matching them is the kinder choice. **The param name is +unchanged** — only the prompt. + +- [ ] **Step 4: Verify the XML parses, the regexes survived, and nothing shares a name** + +```bash +cd ~/workspaces/plasmodb/ApiCommonModel && python3 - <<'PY' +import xml.etree.ElementTree as T, collections +r = T.parse('Model/lib/wdk/model/questions/params/variationParams.xml').getroot() +ps = r.find('paramSet') +names = [c.get('name') for c in ps] +print('params:', names) +dupes = [n for n,c in collections.Counter(names).items() if c > 1] +print('duplicate names:', dupes or 'none') +for sp in ps.iter('stringParam'): + rx = sp.find('regex') + print(sp.get('name'), '->', repr(rx.text if rx is not None else None)) +for e in ps.iter('enumParam'): + print(e.get('name'), 'internals:', [i.find('internal').text for i in e.iter('enumValue')]) +PY +``` + +Expected: **fourteen** params (the eight already there plus your six), **no duplicates**, the +three new `MinPercent*` regexes printing +`'\\d\\d?|100'` (and the pre-existing `MinPercentMinorAlleles` printing `'\\d\\d?'`), and both +enum params listing internals `['80', '60', '40', '20']`. A duplicate name here would mean you +redefined a param the one-group search already provides. + +- [ ] **Step 5: Build to prove the model still loads** + +```bash +cd ~/workspaces/agentic-veupath-dev && bin/veup-build.sh plasmodb wb model +``` + +Expected: completes with no `WdkModelException`. Unreferenced params are legal, so this only +proves the definitions resolve — Task 3 is what exercises them. + +```bash +ssh cedar 'bash -lc "source /var/www/jbrestel.plasmodb.org/etc/setenv && wdkXml -model PlasmoDB"' \ + | grep -E "name='(variation_sample_meta_a|variation_sample_meta_b|MinPercentMajorAlleles|MinPercentMajorAllelesTwo|MinPercentIsolateCallsTwo|ReadFrequencyPercentTwo)'" +``` + +Expected: six matching lines, one per new param, each prefixed with its Java class +(`FilterParamNew`, `StringParam`, `EnumParam`). Note the **single** quotes. + +- [ ] **Step 6: Commit** + +```bash +cd ~/workspaces/plasmodb/ApiCommonModel +git add Model/lib/wdk/model/questions/params/variationParams.xml +git commit -m "Add the two-group sample filters and four thresholds + +Both filters reuse the one-group search's EDA queries; no new SQL. WDK clones a +dependent param's queries per param, which is how the snp original ran two +groups off one query definition. + +No minSelectedCount on either group, deliberately: comparing major alleles +between two groups of one is meaningful, unlike polymorphism within one, and +the snp original set no minimum. + +Only Set B gets 'Two' threshold variants. Set A reuses the unsuffixed +ReadFrequencyPercent and MinPercentIsolateCalls because those are the plugin's +own Set A constants." +``` + +--- + +### Task 3: The query and the question + +**Files:** +- Modify: `ApiCommonModel/Model/lib/wdk/model/questions/queries/variationQueries.xml` +- Modify: `ApiCommonModel/Model/lib/wdk/model/questions/variationQuestions.xml` + +- [ ] **Step 1: Confirm the plugin's ten required params and twelve columns** + +```bash +cd ~/workspaces/plasmodb/ApiCommonWebService +sed -n '18,60p' WSFPlugin/src/main/java/org/apidb/apicomplexa/wsfplugin/highspeedsnpsearch/FindMajorAllelesPlugin.java +``` + +Expected: `PARAM_STRAIN_FILTER_A`/`_B` now reading `variation_sample_meta_a`/`_b` (Task 1), the +Set A constants naming the **unsuffixed** `ReadFrequencyPercent` and `MinPercentIsolateCalls`, +`getRequiredParameterNames()` listing ten, and `getColumns()` listing twelve. If Set A's +constants name suffixed params, stop — the plan's param wiring would be wrong. + +- [ ] **Step 2: Add the process query** + +In `variationQueries.xml`, inside the `VariationsBy` querySet, after `VariationsByGeneIds`: + +```xml + + + + + + + + The Organism defines the species identity of the samples and the genome + against which each sample's variants were called. + + + + + + + + + + + + + + + + + + + + + + + + + + + + +``` + +- [ ] **Step 3: Add the question** + +In `variationQuestions.xml`, after `VariationsByGeneIds`: + +```xml + + + + + + + + + + + + + +
    + + Each sample's sequencing reads are aligned to the reference genome (Organism) + and variants are recorded for each sample based on the Read Frequency + Threshold. Then, scanning locations across the samples in Set A and Set B + separately, the major allele of each set is recorded where it meets that + set's major allele frequency and percent samples with a base call. A location + is returned when the two sets' major alleles differ. + +

    Choosing the two groups: Set A and Set B must differ. Use the + sample characteristics to define each group, for example samples from two + different countries, or two different host phenotypes.

    + +

    Major allele frequency: Among the qualifying calls at a location + within one set, the major allele frequency is the percent carrying the most + common allele. Unlike the within-group searches, 100% is permissible here and + is the most stringent setting: the search identifies each set's major allele + first and then compares the two, so demanding unanimity within a set is a + sharper test rather than an impossible one. Lower the threshold to return + more locations.

    + +

    Read frequency threshold: An allele is called for a sample at a + location if that fraction of the sample's aligned reads support it. Each set + has its own threshold.

    + +

    Percent samples with a base call: A location is only considered + within a set if this fraction of that set's samples have a qualifying call + there.

    + ]]> +     + + + + + + + + + + + + + + + + + + variation_sample_meta_a + variation_sample_meta_b + + +     +``` + +- [ ] **Step 4: Verify both files parse and the question carries all three required blocks** + +```bash +cd ~/workspaces/plasmodb/ApiCommonModel && python3 - <<'PY' +import xml.etree.ElementTree as T +r = T.parse('Model/lib/wdk/model/questions/queries/variationQueries.xml').getroot() +s = r.find('querySet'); print('queries:', [c.get('name') for c in s]) +pq = [c for c in s if c.get('name') == 'VariationsByTwoIsolateGroups'][0] +print('paramRefs:', len(pq.findall('paramRef')), 'wsColumns:', len(pq.findall('wsColumn'))) +r2 = T.parse('Model/lib/wdk/model/questions/variationQuestions.xml').getroot() +print('questions:', [q.get('name') for q in r2.find('questionSet')]) +q = [x for x in r2.iter('question') if x.get('name') == 'VariationsByTwoIsolateGroups'][0] +print('dynAttrs:', len(list(q.iter('columnAttribute')))) +print('summary cols:', len(q.find('attributesList').get('summary').split(','))) +pl = [p for p in q.findall('propertyList') if p.get('name') == 'uniq-value-params'] +print('uniq-value-params:', [v.text for v in pl[0]] if pl else 'MISSING') +PY +``` + +Expected: five queries and five questions; **11 paramRefs** (organism, suffix, wsPath, plus +four per set) and **12 wsColumns**; **10** dynamic attributes; **13** summary columns; and +`uniq-value-params: ['variation_sample_meta_a', 'variation_sample_meta_b']`. `MISSING` there is +the failure this whole task is most likely to produce. + +- [ ] **Step 5: Build and confirm all five searches are in the assembled model** + +```bash +cd ~/workspaces/agentic-veupath-dev && bin/veup-build.sh plasmodb wb model +``` + +Expected: completes with no `WdkModelException`. + +```bash +ssh cedar 'bash -lc "source /var/www/jbrestel.plasmodb.org/etc/setenv && wdkXml -model PlasmoDB"' \ + | grep -oE "VariationsBy\.[A-Za-z]+:" | sort -u +``` + +Expected exactly five: `VariationBySourceId`, `VariationsByGeneIds`, `VariationsByIsolateGroup`, +`VariationsByLocation`, `VariationsByTwoIsolateGroups`. + +- [ ] **Step 6: Commit** + +```bash +cd ~/workspaces/plasmodb/ApiCommonModel +git add Model/lib/wdk/model/questions/queries/variationQueries.xml \ + Model/lib/wdk/model/questions/variationQuestions.xml +git commit -m "Add the VariationsByTwoIsolateGroups search + +Twelve wsColumns and an 11-column results file, both dictated by +FindMajorAllelesPlugin. Params ordered organism, then Set A, then Set B, with +prompt overrides so the two sets' thresholds are distinguishable in the form. + +Carries the uniq-value-params propertyList forbidding Set A = Set B. Its +enforcement is client-side, so no build or service check can detect its +absence; only the rendered form can." +``` + +--- + +### Task 4: Category ontology row + +**Files:** +- Modify: `ApiCommonModel/Model/lib/wdk/ontology/individuals.txt` + +- [ ] **Step 1: Append the row by substitution** + +```bash +cd ~/workspaces/plasmodb/ApiCommonModel +grep "VariationQuestions.VariationBySourceId" Model/lib/wdk/ontology/individuals.txt \ + | sed "s/VariationBySourceId/VariationsByTwoIsolateGroups/g" \ + >> Model/lib/wdk/ontology/individuals.txt +``` + +- [ ] **Step 2: Verify field counts match across all five rows** + +```bash +cd ~/workspaces/plasmodb/ApiCommonModel +awk -F'\t' '/VariationQuestions.Variation/ {print NF" fields "$6}' Model/lib/wdk/ontology/individuals.txt +``` + +Expected: five lines, all `14 fields`, the fifth naming +`VariationQuestions.VariationsByTwoIsolateGroups`. A differing count means a shifted column and +a silently misfiled search — stop rather than patching by hand. + +- [ ] **Step 3: Build with `wb ontology`, NOT `wb model`** + +```bash +cd ~/workspaces/agentic-veupath-dev && bin/veup-build.sh plasmodb wb ontology +``` + +Expected: completes, reporting `categories_merged.owl` saved. `wb model` here leaves the OWL +stale and the search uncategorized, **with no error anywhere**. + +- [ ] **Step 4: Prove the OWL has it under the right parent** + +```bash +ssh cedar 'grep -A3 "individuals.owl#VariationRecordClasses.VariationRecordClass.VariationQuestions.VariationsByTwoIsolateGroups\"" \ + /var/www/jbrestel.plasmodb.org/gus_home/lib/wdk/ontology/categories_merged.owl | grep -c topic_0199' +``` + +Expected: `1`. + +- [ ] **Step 5: Commit** + +```bash +cd ~/workspaces/plasmodb/ApiCommonModel +git add Model/lib/wdk/ontology/individuals.txt +git commit -m "Categorize VariationsByTwoIsolateGroups + +Same placement as the other four variation searches: parent topic_0199, +targetType search, menu + webservice scopes. Completes the five-search port." +``` + +--- + +### Task 5: Browser verification + +**Files:** none. + +- [ ] **Step 1: Mark the logs** + +```bash +cd ~/workspaces/agentic-veupath-dev && bin/veup-logs.sh plasmodb mark twogroups +``` + +- [ ] **Step 2: Open the app and confirm the origin** + +Load `https://jbrestel.plasmodb.org/a/app` (it redirects to the webapp context), then: + +```javascript +({origin: window.location.origin, base: window.location.pathname.split('/')[1]}) +``` + +Expected: origin `https://jbrestel.plasmodb.org`, base `plasmo.jbrestel`. If origin reads +`https://veupathdb.org`, **stop** — the tab bounced to autologin and relative fetches answer for +production. Build all paths from the base you actually got. + +`computer:screenshot` fails on this instance (`Script injection timed out`), including on +known-good pages. Use `javascript_tool` and the service endpoints. + +- [ ] **Step 3: Confirm all five searches are registered** + +```javascript +const b = window.location.pathname.split('/')[1]; +fetch(`/${b}/service/record-types/variation`).then(r=>r.json()) + .then(d=>d.searches.map(s=>s.fullName).filter(n=>n.startsWith('VariationQuestions'))) +``` + +Expected: all five, including `VariationQuestions.VariationsByTwoIsolateGroups`. + +- [ ] **Step 4: Confirm the form's shape and both filters populate** + +```javascript +const b = window.location.pathname.split('/')[1]; +const q = await fetch(`/${b}/service/record-types/variation/searches/VariationsByTwoIsolateGroups?expandParams=true`).then(r=>r.json()); +const ps = q.searchData.parameters; +JSON.stringify({ + names: ps.map(p=>p.name), + prompts: ps.map(p=>p.displayName), + filters: ps.filter(p=>p.type==='filter').map(p=>({name:p.name, nodes:p.ontology.length, min:p.minSelectedCount})) +}, null, 1) +``` + +Expected: eleven params; the two filters are `variation_sample_meta_a` and +`variation_sample_meta_b`, each with **27** ontology nodes and **no** `minSelectedCount`; and +the Set A / Set B prompts are distinct (not two identical "Read frequency threshold" labels). + +- [ ] **Step 5: The disjoint-groups run — the search's definition** + +`country` splits the Pf samples cleanly: 147 French Guiana, 69 Senegal. Run with Set A = +Senegal, Set B = French Guiana, thresholds at their defaults (80% RFT, 80 major allele, 20 +percent called). The `country` ontology term is `VAR_8e68b3e5`; a filter value looks like: + +```javascript +JSON.stringify({filters:[{field:"VAR_8e68b3e5", type:"string", isRange:false, + value:["Senegal"], includeUnknown:false}]}) +``` + +Submit via `/reports/standard` requesting attributes +`["primary_key","MajorAlleleA","MajorAlleleB"]`, then assert: + +```javascript +const bad = records.filter(r => r.attributes.MajorAlleleA === r.attributes.MajorAlleleB); +({total: meta.totalCount, sampled: records.length, violations: bad.length}) +``` + +Expected: a non-empty result set and **`violations: 0`**. `MajorAlleleA != MajorAlleleB` *is* +the search's definition, so a single violation means the comparison is broken — this is a +stronger assertion than any row count. + +- [ ] **Step 6: The symmetry check** + +Swap the groups: Set A = French Guiana, Set B = Senegal. Expect an **identical `totalCount`**, +because "the two major alleles disagree" is symmetric. A differing count means Set B's +thresholds are not applied the way Set A's are — exactly the copy-paste asymmetry that doubled +params invite. + +Caveat to record in the report: this tests the *threshold* plumbing. It would also pass +trivially if both sets read the same `readFreq` files, which on this haploid site they do. + +- [ ] **Step 7: The `uniq-value-params` check — browser only** + +In the rendered form, set both groups to the same value (Senegal in each). The form should +refuse to submit. If it submits, the property did not take effect; report it rather than working +around it, since nothing else can detect this. + +- [ ] **Step 8: Confirm an ID resolves** + +Take one returned ID (form `Variant__`), navigate to +`//app/record/variation/`, and confirm the record page renders without error. + +- [ ] **Step 9: Category tree** + +```javascript +const b = window.location.pathname.split('/')[1]; +const c = await fetch(`/${b}/service/ontologies/Categories`).then(r=>r.json()); +const out=[]; +(function walk(n,parent){const p=n.properties||{};const nm=(p.name||[])[0]; + if(nm && nm.startsWith('VariationQuestions')) out.push({name:nm.replace('VariationQuestions.',''), parent}); + (n.children||[]).forEach(ch=>walk(ch,(p['EuPathDB alternative term']||p.label||[])[0]||parent)); +})(c.tree,'ROOT'); +JSON.stringify(out,null,1) +``` + +Expected: all five searches, each with parent `"Genetic variation"`. + +- [ ] **Step 10: Logs** + +```bash +cd ~/workspaces/agentic-veupath-dev && bin/veup-logs.sh plasmodb since twogroups --quiet +cd ~/workspaces/agentic-veupath-dev && bin/veup-logs.sh plasmodb since twogroups 2>&1 \ + | grep -oE "hsssGenerate[A-Za-z]+|findMajorAlleles" | sort | uniq -c +``` + +Expected: error logs `silent:`, and `hsssGenerateMajorAllelesScript` appearing once per run. + +**Any ERROR lines caused by your own malformed service requests must be reported as +self-inflicted, not as "the logs were clean."** + +- [ ] **Step 11: Report** + +Report: the five registered searches; the eleven params with their prompts and the two filters' +node counts; for both runs the total count and the violation count; whether the form refused +A = B; one ID that resolved; the category-tree parents; and the log verdict distinguishing +self-inflicted errors from real ones. + +--- + +## Out of scope + +- **`NgsSnpsByTwoIsolateGroupsWiz`** — a sixth search (PlasmoDB/UniDB only) driven by the + `*_wiz` params, which sit in the same commented-out region as the params this plan replaces. + Whether the wizard flow is still wanted is a product question. **With this plan the + five-search port is complete.** +- **Deleting the dead snp XML.** After this, `snpParams.xml` and the commented-out snp regions + of `sharedParams.xml` have no remaining consumer, but removal has its own blast radius + (`recordParams.xml`, `spanQuestions.xml`, `SnpsBySpanLogic`). +- **The Set B null-check asymmetry** in `FindMajorAllelesPlugin`. +- **`ReadFrequencyPercent` as a functional parameter.** All four `readFreq*` directories hold + identical data on haploid organisms; both read-frequency params select their directory + correctly, and changing either will not change results. Not a defect in this search. diff --git a/docs/superpowers/plans/2026-08-06-genetic-variation-searches-port.md b/docs/superpowers/plans/2026-08-06-genetic-variation-searches-port.md new file mode 100644 index 000000000..743da518e --- /dev/null +++ b/docs/superpowers/plans/2026-08-06-genetic-variation-searches-port.md @@ -0,0 +1,1785 @@ +# Genetic Variation Searches Port — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Port the four remaining Genetic Variation searches — `GenesByNgsSnps`, +`SequencesByPloidy`, `GenesByCopyNumber`, `GenesByCopyNumberComparison` — onto the merged +dnaseq/EDA world, so all eight dnaseq searches work and are categorized together. + +**Architecture:** `GenesByNgsSnps` is uncommented and repointed from the absent `snpParams` +paramSet to its live `variationParams` twins; its HSSS plugin was already fixed by the +2026-08-05 plumbing work. The three CNV searches are rebuilt on corrected copy-number tables +that drop the dead `PANIO_p`/`study.Input` join and key on an EDA sample stable ID. Those +tables land transitionally in `apidbtuning` (buildable now via Jenkins) and permanently in +`webready/*.psql` (next workflow run). + +**Tech Stack:** WDK model XML (questions, queries, params, ontology), EuPathDB tuningManager +XML, PostgreSQL, the `wb` build wrapper, Claude in Chrome for live QA. + +**Spec:** `ApiCommonModel/docs/superpowers/specs/2026-08-06-genetic-variation-searches-port-design.md` + +--- + +## How "TDD" works here + +There is no unit-test harness for WDK model XML. The test-first discipline still applies, +in this form — **every task establishes a failing observation before changing anything**: + +| Instead of | Do this | +|---|---| +| write a failing test | run the build / query / service endpoint and **capture the current failure or absence** | +| watch it fail | confirm the exact error text or the missing item | +| implement | make the edit | +| watch it pass | re-run the *same* command and confirm the failure is gone | + +Never skip the "before" observation. Three of these four searches currently **build fine and +silently return nothing**, which is exactly the failure mode a "did it build?" check misses. + +### The app lives under a context path — `/service` alone is a 404 + +Verified 2026-08-06 from an authenticated tab. The app is served at +`https://jbrestel.plasmodb.org/plasmo.jbrestel/app`, so: + +| fetch | result | +|---|---| +| `/service/record-types/transcript` | **404**, `text/html` | +| `/plasmo.jbrestel/service/record-types/transcript` | **200**, `application/json` | + +Every `javascript_tool` snippet in this plan therefore derives the base from the current +location rather than hardcoding it: + +```javascript +const BASE = window.location.pathname.replace(/\/app.*$/, ''); // -> "/plasmo.jbrestel" +``` + +Keep the origin guard as well. The two catch different failures: the origin guard catches +an unauthenticated tab that has been redirected to **production** (where the fetch would +succeed and answer for the wrong site), and the BASE derivation catches the 404. A bare +`/service` fetch throws at `.json()` rather than returning empty, so it fails loudly — but +only if you do not wrap it in a try/catch that swallows it. + +### `wdkQuery -showQuery` does not work on these queries — use `-showParams` + +Discovered during Task 1 and verified against a **known-good** search. Any query with a +dependent `filterParam` makes the `wdkQuery` CLI fail before it renders SQL: + +``` +ERROR - org.gusdb.wdk.model.test.ParamValuesFactory:84 - Unable to populate param values set with defaults +``` + +`VariationsBy.VariationsByIsolateGroup` — one of the five searches that work in production +today — fails identically. It is a CLI auto-default limitation, **not** a defect in the +query, so do not treat it as one and do not try to fix it. + +This affects every query in this plan that uses `variation_sample_meta` or +`cnv_sample_meta`: Tasks 1, 9, 10 and 12. + +**Use instead:** + +- **`-showParams`** to prove the query resolves and its params are correctly typed. This + works, and is the "did my XML wire up" check. +- **The model XML itself** as the SQL to run in psql. This is normally *not* safe advice — + the CLAUDE.md rule is that raw model XML is not what executes, because + `presenterInjectTemplates` expands `-- TEMPLATE_ANCHOR` into per-dataset `UNION` branches. + It is safe **here specifically**: all three CNV queries were checked and contain **zero** + `TEMPLATE_ANCHOR` occurrences, so the XML SQL and the assembled SQL are identical. If you + add a template anchor to any of them, this shortcut stops being valid. + +## Environment + +All commands run from `/home/jbrestel/workspaces/agentic-veupath-dev` unless stated. +Source edits are in `/home/jbrestel/workspaces/plasmodb//`, already on branch +`dnaseq-merge-experiments`. Mutagen carries edits to the remote; **do not** run builds +locally. + +| Thing | Value | +|---|---| +| build | `bin/veup-build.sh plasmodb wb model` / `... wb ontology` | +| logs | `bin/veup-logs.sh plasmodb mark