From 512268b2bf280e791c21dd389dc2f58962194fff Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 5 Mar 2026 18:54:31 +0000 Subject: [PATCH 1/6] Initial plan From 2eab5fac85b915abdf15b02a7ddbae5a3ebcda32 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 5 Mar 2026 19:05:21 +0000 Subject: [PATCH 2/6] Refactor: modularize tdbsumstat export command into separate modules with tests Co-authored-by: bruno-ariano <26384813+bruno-ariano@users.noreply.github.com> --- scripts/create_metadata.py | 69 ++++++++ scripts/fix_json.py | 55 ++++++ scripts/generate_table_cell_sumstat.py | 22 +++ tdbsumstat/cli/export.py | 20 ++- tdbsumstat/cli/export/__init__.py | 4 + tdbsumstat/cli/export/command.py | 141 ++++++++++++++++ tdbsumstat/cli/export/helpers.py | 45 +++++ tdbsumstat/cli/export/locusbreaker.py | 124 ++++++++++++++ tdbsumstat/cli/export/metadata.py | 135 +++++++++++++++ tdbsumstat/cli/export/regions.py | 50 ++++++ tdbsumstat/cli/export/snp.py | 72 ++++++++ tdbsumstat/cli/export/traits.py | 54 ++++++ tdbsumstat/utils/create_metadata.py | 71 +------- tdbsumstat/utils/fix_json.py | 57 +------ .../utils/generate_table_cell_sumstat.py | 24 +-- tests/__init__.py | 0 tests/conftest.py | 88 ++++++++++ tests/test_export.py | 147 +++++++++++++++++ tests/test_harmonize.py | 156 ++++++++++++++++++ tests/test_utils.py | 121 ++++++++++++++ 20 files changed, 1308 insertions(+), 147 deletions(-) create mode 100755 scripts/create_metadata.py create mode 100644 scripts/fix_json.py create mode 100755 scripts/generate_table_cell_sumstat.py create mode 100644 tdbsumstat/cli/export/__init__.py create mode 100644 tdbsumstat/cli/export/command.py create mode 100644 tdbsumstat/cli/export/helpers.py create mode 100644 tdbsumstat/cli/export/locusbreaker.py create mode 100644 tdbsumstat/cli/export/metadata.py create mode 100644 tdbsumstat/cli/export/regions.py create mode 100644 tdbsumstat/cli/export/snp.py create mode 100644 tdbsumstat/cli/export/traits.py create mode 100644 tests/__init__.py create mode 100644 tests/conftest.py create mode 100644 tests/test_export.py create mode 100644 tests/test_harmonize.py create mode 100644 tests/test_utils.py diff --git a/scripts/create_metadata.py b/scripts/create_metadata.py new file mode 100755 index 0000000..8c93977 --- /dev/null +++ b/scripts/create_metadata.py @@ -0,0 +1,69 @@ +import json +from collections import defaultdict +input_path = "/lustre/scratch124/humgen/projects_v2/cardinal_analysis/analysis/core_dataset/freeze3/tiledbs/Tensor/TileDB_f3_UKB_cis_t2_metadata.json" +output_path = input_path.replace(".json", "_fixed.json") + +objs = [] +with open(input_path) as f: + text = f.read() + +decoder = json.JSONDecoder() +idx = 0 +while idx < len(text): + obj, idx = decoder.raw_decode(text, idx) + objs.append(obj) + +with open(output_path, "w") as f: + json.dump(objs, f, indent=2) + +print(f"✅ Salvaged {len(objs)} metadata records into {output_path}") + + +fixed_path = "/lustre/scratch124/humgen/projects_v2/cardinal_analysis/analysis/core_dataset/freeze3/tiledbs/Tensor/TileDB_f3_UKB_cis_t2_metadata_fixed.json" +output_path = fixed_path.replace("_fixed.json", "_merged.json") + +with open(fixed_path) as f: + metadata_records = json.load(f) + +# Create a merged dictionary +merged_metadata = { + "file_path": [], + "trait": [], + "celltype": [] +} + +# For storing per-cell/chromosome info if present +cell_chrom_data = defaultdict(lambda: defaultdict(list)) + +for record in metadata_records: + # Merge file paths + if "file_path" in record and record["file_path"] not in merged_metadata["file_path"]: + merged_metadata["file_path"].append(record["file_path"]) + # Merge traits + if len(record["trait"]) > 0: + for t in record["trait"]: + if t not in merged_metadata["trait"]: + merged_metadata["trait"].append(t) + # Merge celltypes and their per-chromosome data + if len(record["celltype"]) > 0: + for cell in record["celltype"]: + if cell not in merged_metadata["celltype"]: + merged_metadata["celltype"].append(cell) + # Merge chromosome-level info if exists + if cell in record: + for chrom, genes in record[cell].items(): + cell_chrom_data[cell][chrom] = {} + #existing_genes = {g for g in cell_chrom_data[cell][chrom]} + for g in genes: + #if g not in existing_genes: + cell_chrom_data[cell][chrom][g] = genes[g] + +# Add per-cell/chromosome info +for cell, chrom_dict in cell_chrom_data.items(): + merged_metadata[cell] = chrom_dict + +# Save as a single JSON object +with open(output_path, "w") as f: + json.dump(merged_metadata, f, indent=2) + +print(f"✅ Merged metadata saved to {output_path}") diff --git a/scripts/fix_json.py b/scripts/fix_json.py new file mode 100644 index 0000000..ef83c6c --- /dev/null +++ b/scripts/fix_json.py @@ -0,0 +1,55 @@ +import json +import re +from collections import defaultdict + +# === STEP 1: Read the entire string from file === +with open("/project/cardinal/QTLs/freeze3/TileDBs/TileDB_metanalyses_gh_celltype2_freeze3_metadata.json", "r", encoding="utf-8") as f: + data_str = f.read() + +# === STEP 2: Split based on the start of each JSON fragment === +# We capture the first '{' before each file_path +fragments = re.findall(r'(\{[^{]*"file_path":.*?)(?=\{[^{]*"file_path":|\Z)', data_str, flags=re.S) + +print(f"🧩 Found {len(fragments)} fragments") + +# === STEP 3: Parse each fragment === +parsed_fragments = [] +for i, frag in enumerate(fragments): + try: + parsed_fragments.append(json.loads(frag)) + except json.JSONDecodeError as e: + print(f"⚠️ Fragment {i} failed to parse at char {e.pos}: {e}") + print(frag[:300] + "...") + # optional: skip or raise + continue + +# === STEP 4: Merge fragments as before === +merged = { + "trait": [], + "celltype": [], +} +celltype_data = defaultdict(lambda: defaultdict(dict)) + +for frag in parsed_fragments: + if "trait" in frag and isinstance(frag["trait"], list): + merged["trait"].extend(frag["trait"]) + if "celltype" in frag and isinstance(frag["celltype"], list): + merged["celltype"].extend(frag["celltype"]) + for key, val in frag.items(): + if key in ("trait", "celltype", "CELL", "file_path"): + continue + for num, genes in val.items(): + for gene_id, gene_data in genes.items(): + celltype_data[key][num][gene_id] = gene_data + +for celltype, numbers in celltype_data.items(): + merged[celltype] = numbers + +merged["trait"] = list(dict.fromkeys(merged["trait"])) +merged["celltype"] = list(dict.fromkeys(merged["celltype"])) + +# === STEP 5: Save merged JSON === +with open("metadata_merged.json", "w", encoding="utf-8") as out_f: + json.dump(merged, out_f, indent=2) + +print("✅ Merged JSON written to metadata_merged.json") diff --git a/scripts/generate_table_cell_sumstat.py b/scripts/generate_table_cell_sumstat.py new file mode 100755 index 0000000..43a4409 --- /dev/null +++ b/scripts/generate_table_cell_sumstat.py @@ -0,0 +1,22 @@ +import pandas as pd +import tiledb +rows = [] +tdb = tiledb.open('/lustre/scratch124/humgen/projects_v2/cardinal_analysis/analysis/core_dataset/freeze3/tiledbs/Tensor/TileDB_f3_UKB_cis_t2', 'w') +# Loop over all celltypes in metadata +for celltype in merged_metadata["celltype"]: + if celltype in merged_metadata: # make sure the key exists in dict + groups = merged_metadata[celltype] + for group, values in groups.items(): # e.g. group "11" + for gene_id in values: + rows.append({ + "CHR": group, + "CELL": celltype, + "GENE": gene_id, + "N": values[gene_id]["N"], + "ACAT": values[gene_id]["ACAT"], + "PHENOVAR": values[gene_id]["PHENO_VAR"], + }) +df = pd.DataFrame(rows) + +tdb.meta["metadata"] = json.dumps(merged_metadata) +df.to_csv("/lustre/scratch124/humgen/projects_v2/cardinal_analysis/analysis/core_dataset/freeze3/tiledbs/Tensor/Bangladeshi/metadata_gh_bangladeshi_f3_celltype1_acat.csv",index = False) \ No newline at end of file diff --git a/tdbsumstat/cli/export.py b/tdbsumstat/cli/export.py index b6ea133..dad210b 100755 --- a/tdbsumstat/cli/export.py +++ b/tdbsumstat/cli/export.py @@ -1,3 +1,21 @@ +# This file is superseded by the tdbsumstat/cli/export/ package. +# Python loads the package (directory with __init__.py) in preference to this file. +# It is kept for reference only; all logic now lives in the export/ sub-modules: +# export/helpers.py – common TileDB helpers +# export/snp.py – SNP-based export +# export/regions.py – region-based export +# export/locusbreaker.py – locusbreaker export +# export/metadata.py – metadata export / recompute +# export/traits.py – trait-based export +# export/command.py – CLI command definition and routing +# export/__init__.py – re-exports the `export` command + +from tdbsumstat.cli.export.command import export # noqa: F401 + +# --------------------------------------------------------------------------- +# Legacy code kept below for historical reference only +# --------------------------------------------------------------------------- + import tiledb import click import cloup @@ -14,7 +32,7 @@ Query TileDB database and export data. """ -@cloup.command("export", no_args_is_help=True, help=help_doc) +@cloup.command("export_legacy", no_args_is_help=True, help=help_doc) @cloup.option_group( "Options for querying specific chromosomes, cells, genes or positions in the TileDB", cloup.option("--uri-path", default = None, type=str, help = "path of TileDB"), diff --git a/tdbsumstat/cli/export/__init__.py b/tdbsumstat/cli/export/__init__.py new file mode 100644 index 0000000..e229e84 --- /dev/null +++ b/tdbsumstat/cli/export/__init__.py @@ -0,0 +1,4 @@ +"""Export package – re-exports the ``export`` CLI command.""" +from tdbsumstat.cli.export.command import export + +__all__ = ["export"] diff --git a/tdbsumstat/cli/export/command.py b/tdbsumstat/cli/export/command.py new file mode 100644 index 0000000..12c2386 --- /dev/null +++ b/tdbsumstat/cli/export/command.py @@ -0,0 +1,141 @@ +"""CLI command definition for the ``export`` subcommand. + +This module wires CLI options to the individual export handler functions +defined in the sibling modules (snp, regions, locusbreaker, metadata, traits). +""" +import click +import cloup + +from tdbsumstat.cli.export.helpers import open_tiledb_and_load_metadata +from tdbsumstat.cli.export.locusbreaker import export_with_locusbreaker +from tdbsumstat.cli.export.metadata import export_metadata, recompute_metadata +from tdbsumstat.cli.export.regions import export_by_regions +from tdbsumstat.cli.export.snp import export_by_snp +from tdbsumstat.cli.export.traits import export_by_traits + +help_doc = """ +Query TileDB database and export data. +""" + + +@cloup.command("export", no_args_is_help=True, help=help_doc) +@cloup.option_group( + "Options for querying specific chromosomes, cells, genes or positions in the TileDB", + cloup.option("--uri-path", default=None, type=str, help="path of TileDB"), + cloup.option("--table-regions", default=None, type=str, help="Regions to interrogate from a table"), + cloup.option("--trait-list", default=None, type=str, help="List of entire traits to filter"), + cloup.option("--attr", default="P,SNPID,EAF,BETA,SE", type=str, help="Attributes to output"), + cloup.option("--export-meta", is_flag=True, default=False, type=str, help="Get metadata from TileDB"), + cloup.option("--mac", default=0, type=int, help="Filter for MAC when recomputing metadata"), + cloup.option( + "--recompute-meta", + is_flag=True, + default=False, + type=str, + help="Recompute metadata after applying filters (Does not modify data within the TileDB)", + ), + cloup.option( + "--snp", + default=None, + type=str, + help="List of SNPs to interrogate taken from a txt file. Please check README for details on the format of this file", + ), + cloup.option("--batch-name", default=None, type=str, help="Name of the batch"), +) +@cloup.option_group( + "Options for Locusbreaker", + cloup.option("--locusbreaker", is_flag=True, type=bool, default=False, help="Option to run locusbreaker"), + cloup.option( + "--hole-lb", + default=250000, + type=int, + help="Minimum base-pair distance between SNPs in different loci (default: 250000)", + ), + cloup.option( + "--maf-lb", + default=0.001, + type=float, + help="The MAF to filter the TILEDB before locusbreaker is run", + ), + cloup.option( + "--locus-max-size-lb", + default=3000000, + type=float, + help="The maximum size allowed for the locus. Default: 1Mb", + ), + cloup.option( + "--cis-trans-lb", + default="cis", + type=str, + help="If locusbreaker run on cis or trans QLTs", + ), + cloup.option("--table-lb", default=None, type=str, help="Path of the table to provide"), + cloup.option("--type-sumstat", default=None, type=str, help="Type of summary data"), +) +@cloup.option_group( + "Options for output", + cloup.option( + "--out", + default="out", + type=str, + help="Output path with file name where results will be stored", + ), +) +@click.pass_context +def export( + ctx, + uri_path: str, + type_sumstat: str, + table_regions: str, + trait_list: str, + mac: int, + attr: str, + snp: str, + export_meta: bool, + recompute_meta: bool, + locusbreaker: bool, + maf_lb: float, + cis_trans_lb: str, + table_lb: str, + hole_lb: int, + out: str, + locus_max_size_lb: int, + batch_name: str, +): + """Route the export command to the appropriate handler based on CLI flags.""" + if snp: + tiledb_export, _df_meta = open_tiledb_and_load_metadata(uri_path, type_sumstat) + export_by_snp(tiledb_export, snp, attr, type_sumstat, out) + tiledb_export.close() + + elif table_regions: + tiledb_export, _df_meta = open_tiledb_and_load_metadata(uri_path, type_sumstat) + export_by_regions(tiledb_export, table_regions, attr, type_sumstat, out) + tiledb_export.close() + + elif locusbreaker: + _tiledb_export, df_meta = open_tiledb_and_load_metadata(uri_path, type_sumstat) + _tiledb_export.close() + export_with_locusbreaker( + uri_path=uri_path, + df_meta=df_meta, + table_lb=table_lb, + maf_lb=maf_lb, + hole_lb=hole_lb, + locus_max_size_lb=locus_max_size_lb, + cis_trans_lb=cis_trans_lb, + type_sumstat=type_sumstat, + out=out, + batch_name=batch_name, + ) + + elif export_meta: + export_metadata(uri_path, type_sumstat, out) + + elif recompute_meta: + tiledb_export, df_meta = open_tiledb_and_load_metadata(uri_path, type_sumstat) + recompute_metadata(tiledb_export, df_meta, trait_list, type_sumstat, mac, out, batch_name) + tiledb_export.close() + + else: + export_by_traits(uri_path, trait_list, attr, type_sumstat, out, batch_name) diff --git a/tdbsumstat/cli/export/helpers.py b/tdbsumstat/cli/export/helpers.py new file mode 100644 index 0000000..4ca5ddc --- /dev/null +++ b/tdbsumstat/cli/export/helpers.py @@ -0,0 +1,45 @@ +"""Common TileDB helper functions for export operations.""" +import json + +import polars as pl +import tiledb + + +def open_tiledb_and_load_metadata(uri_path: str, type_sumstat: str): + """Open TileDB array and load metadata as a Polars DataFrame. + + Parameters + ---------- + uri_path : str + Path to the TileDB array. + type_sumstat : str + Type of summary statistics: "gwas" or "qtl". + + Returns + ------- + tuple[tiledb.Array, pl.DataFrame] + Open TileDB array (read mode) and a Polars DataFrame with metadata. + """ + tiledb_export = tiledb.open(uri_path, mode="r") + metadata = json.loads(tiledb_export.meta["merged_metadata"]) + rows = [] + if type_sumstat == "qtl": + for cell in metadata["CELL"]: + for chrom, genes in metadata[cell].items(): + for gene, stats in genes.items(): + rows.append({ + "CELL": cell, + "CHR": chrom, + "GENE": gene, + **stats + }) + df_meta = pl.DataFrame(rows) + df_meta = df_meta.with_columns(pl.col("CHR").cast(pl.UInt16)) + else: + for trait in metadata["trait"]: + rows.append({ + "TRAIT": trait, + **metadata[trait] + }) + df_meta = pl.DataFrame(rows) + return tiledb_export, df_meta diff --git a/tdbsumstat/cli/export/locusbreaker.py b/tdbsumstat/cli/export/locusbreaker.py new file mode 100644 index 0000000..6210d7e --- /dev/null +++ b/tdbsumstat/cli/export/locusbreaker.py @@ -0,0 +1,124 @@ +"""Locusbreaker-based export from TileDB.""" +import os +import random + +import pandas as pd +import polars as pl +import tiledb + +from tdbsumstat.utils.locusbreaker_plpl import locusbreaker_plpl + + +def _query_tiledb_for_locusbreaker( + uri_path: str, + chrom: int, + type_sumstat: str, + trait: str = None, + cell: str = None, + gene: str = None, +) -> "pa.Table": + """Query TileDB for a specific chromosome/trait combination. + + Returns a PyArrow table for use with locusbreaker_plpl. + """ + with tiledb.open(uri_path, mode="r") as tiledb_data: + if type_sumstat == "gwas": + return tiledb_data.query(dims=["CHR", "TRAIT", "POS"]).df[chrom, trait, :] + else: + return tiledb_data.query(dims=["CHR", "CELL", "GENE", "POS"], return_arrow=True).df[ + chrom, cell, gene, : + ] + + +def export_with_locusbreaker( + uri_path: str, + df_meta: pl.DataFrame, + table_lb: str, + maf_lb: float, + hole_lb: int, + locus_max_size_lb: int, + cis_trans_lb: str, + type_sumstat: str, + out: str, + batch_name: str, + pvalue_sig: float = 5e-8, + pvalue_limit: float = 5e-6, +) -> None: + """Run locusbreaker on TileDB data and export loci intervals and segments. + + Parameters + ---------- + uri_path : str + Path to the TileDB array. + df_meta : pl.DataFrame + Polars DataFrame with merged metadata (used by locusbreaker_plpl). + table_lb : str + Path to a CSV table with CHR, TRAIT (and optionally SIG, LIM columns). + maf_lb : float + MAF filter applied before locusbreaker. + hole_lb : int + Minimum base-pair distance to separate loci. + locus_max_size_lb : int + Maximum allowed locus size in base pairs. + cis_trans_lb : str + Filter type: "cis" or "trans" (for QTL data). + type_sumstat : str + Type of summary statistics: "gwas" or "qtl". + out : str + Output file prefix. + batch_name : str + Batch identifier appended to output file names. + pvalue_sig : float + P-value threshold for significant SNPs (default: 5e-8). + pvalue_limit : float + P-value threshold for locus boundary definition (default: 5e-6). + """ + print("Starting LocusBreaker") + traits = pd.read_csv(table_lb) + traits = traits.astype({"CHR": "int16"}) + + if not batch_name: + batch_name = random.randint(1, 10000000) + + for _index, trait in traits.iterrows(): + if "SIG" in traits.columns: + pvalue_sig = trait["SIG"] + pvalue_limit = trait["LIM"] + + if type_sumstat == "gwas": + query = _query_tiledb_for_locusbreaker(uri_path, trait["CHR"], type_sumstat, trait=trait["TRAIT"]) + else: + cell, genes = trait["TRAIT"].split(";") + query = _query_tiledb_for_locusbreaker(uri_path, trait["CHR"], type_sumstat, cell=cell, gene=genes) + + result = locusbreaker_plpl( + query, + maf=maf_lb, + pvalue_sig=pvalue_sig, + pvalue_limit=pvalue_limit, + locus_max_size=locus_max_size_lb, + hole_size=hole_lb, + cis_trans_lb=cis_trans_lb, + type_sumstat=type_sumstat, + metadata=df_meta, + ) + + if not len(result) == 0 and not result[0].empty: + if result and isinstance(result[0], pd.DataFrame) and not result[0].shape[0] == 0: + interval = result[0] + segments = result[1] + + write_header_interval = not os.path.exists(f"{out}_batch_{batch_name}_interval.csv") + write_header_segment = not os.path.exists(f"{out}_batch_{batch_name}_segment.csv") + interval.to_csv( + f"{out}_batch_{batch_name}_interval.csv", + mode="a", + index=False, + header=write_header_interval, + ) + segments.to_csv( + f"{out}_batch_{batch_name}_segment.csv", + mode="a", + index=False, + header=write_header_segment, + ) diff --git a/tdbsumstat/cli/export/metadata.py b/tdbsumstat/cli/export/metadata.py new file mode 100644 index 0000000..817ccb7 --- /dev/null +++ b/tdbsumstat/cli/export/metadata.py @@ -0,0 +1,135 @@ +"""Metadata export and recomputation from TileDB.""" +import json + +import pandas as pd +import polars as pl +import tiledb + +from tdbsumstat.utils import acat_optimized + + +def export_metadata(uri_path: str, type_sumstat: str, out: str) -> None: + """Export merged metadata from TileDB to a CSV file. + + Parameters + ---------- + uri_path : str + Path to the TileDB array. + type_sumstat : str + Type of summary statistics: "gwas" or "qtl". + out : str + Output file prefix (``{out}_meta.csv`` will be created). + """ + tiledb_db = tiledb.open(uri_path, mode="r") + tiledb_meta = json.loads(tiledb_db.meta["merged_metadata"]) + rows = [] + + if type_sumstat == "qtl": + for cell_type in tiledb_meta["CELL"]: + samples = tiledb_meta.get(cell_type, {}) + for sample_id, genes in samples.items(): + for gene_id, metrics in genes.items(): + rows.append({ + "CELL": cell_type, + "CHR": sample_id, + "GENE": gene_id, + **metrics, + }) + else: + for trait in tiledb_meta["traits"]: + samples = tiledb_meta.get(trait, {}) + for chrom, genes in samples.items(): + for gene_id, metrics in genes.items(): + rows.append({ + "TRAIT": trait, + "CHR": chrom, + **metrics, + }) + + df = pd.DataFrame(rows) + df.to_csv(f"{out}_meta.csv", index=False) + tiledb_db.close() + + +def recompute_metadata( + tiledb_export: tiledb.Array, + df_meta: pl.DataFrame, + trait_list: str, + type_sumstat: str, + mac: int, + out: str, + batch_name: str, +) -> None: + """Recompute metadata statistics after applying MAC filter and save to CSV. + + This does **not** modify data stored inside the TileDB array. + + Parameters + ---------- + tiledb_export : tiledb.Array + Open TileDB array in read mode. + df_meta : pl.DataFrame + Polars DataFrame with merged metadata. + trait_list : str + Path to a CSV file with traits/cells to recompute. + type_sumstat : str + Type of summary statistics: "gwas" or "qtl". + mac : int + Minimum minor allele count filter. + out : str + Output file prefix. + batch_name : str + Batch identifier appended to output file names. + """ + trait_list_pd = pd.read_csv(trait_list) + + for _record, trait in trait_list_pd.iterrows(): + if type_sumstat == "gwas": + tiledb_query = tiledb_export.query().df[int(trait["CHR"]), trait["TRAIT"].to_string(), :] + n = df_meta.filter(pl.col("TRAIT") == trait["TRAIT"]).select("N")["N"][0] + else: + print(trait["CELL"]) + tiledb_query = tiledb_export.query().df[int(trait["CHR"]), trait["CELL"], :, :] + n = df_meta.filter(pl.col("CELL") == trait["CELL"]).select("N")["N"][0] + print(n) + + if "N" not in tiledb_query.columns: + tiledb_query["N"] = n + print(tiledb_query) + + tiledb_query_pl = pl.from_pandas(tiledb_query) + tiledb_query_pl = tiledb_query_pl.with_columns( + (2 * pl.col("N") * pl.min_horizontal("EAF", (1 - pl.col("EAF")))).alias("MAC") + ).filter(pl.col("MAC") > mac) + + if type_sumstat == "gwas": + chr_gene_agg = tiledb_query_pl.group_by(["CHR", "TRAIT"]).agg([ + pl.col("P") + .map_batches( + lambda s: pl.Series([acat_optimized(s)]), + return_dtype=pl.Float64, + ) + .alias("ACAT_LIST"), + pl.col("N").first().alias("N"), + pl.min("P").alias("MIN_P"), + ]) + chr_gene_agg = chr_gene_agg.with_columns( + pl.col("ACAT_LIST").list.first().alias("ACAT") + ) + else: + chr_gene_agg = tiledb_query_pl.group_by(["CHR", "CELL", "GENE"]).agg([ + pl.col("P") + .map_batches( + lambda s: pl.Series([acat_optimized(s)]), + return_dtype=pl.Float64, + ) + .alias("ACAT_LIST"), + pl.col("N").first().alias("N"), + pl.min("P").alias("MIN_P"), + ]) + chr_gene_agg = chr_gene_agg.with_columns( + pl.col("ACAT_LIST").list.first().alias("ACAT"), + ).drop("ACAT_LIST") + + chr_gene_agg_pd = chr_gene_agg.to_pandas() + chr_gene_agg_pd.to_csv(f"{out}_batch_{batch_name}_metadata.csv", mode="a", index=False) diff --git a/tdbsumstat/cli/export/regions.py b/tdbsumstat/cli/export/regions.py new file mode 100644 index 0000000..3fdc047 --- /dev/null +++ b/tdbsumstat/cli/export/regions.py @@ -0,0 +1,50 @@ +"""Region-based export from TileDB.""" +import pandas as pd +import tiledb + + +def export_by_regions( + tiledb_export: tiledb.Array, + table_regions: str, + attr: str, + type_sumstat: str, + out: str, +) -> None: + """Query TileDB by genomic regions table and export results to CSV. + + Parameters + ---------- + tiledb_export : tiledb.Array + Open TileDB array in read mode. + table_regions : str + Path to a CSV file with columns CHR, START, END, TRAIT. + attr : str + Comma-separated list of attributes to export. + type_sumstat : str + Type of summary statistics: "gwas" or "qtl". + out : str + Output file path. + """ + pd_region = pd.read_csv(table_regions) + counter_nonempty_region = 0 + + for _ind, row in pd_region.iterrows(): + if type_sumstat == "gwas": + trait = row["TRAIT"] + region = tiledb_export.query( + dims=["CHR", "POS", "TRAIT"], + attrs=attr.split(","), + ).df[int(row["CHR"]), trait, int(row["START"]):int(row["END"])] + else: + cell, gene = row["TRAIT"].split(":") + region = tiledb_export.query( + dims=["CHR", "POS", "CELL", "GENE"], + attrs=attr.split(","), + ).df[int(row["CHR"]), cell, gene, int(row["START"]):int(row["END"])] + + if len(region) > 0: + if counter_nonempty_region == 0: + region.to_csv(out, mode="a", index=False, header=True) + counter_nonempty_region += 1 + else: + region.to_csv(out, mode="a", index=False, header=False) diff --git a/tdbsumstat/cli/export/snp.py b/tdbsumstat/cli/export/snp.py new file mode 100644 index 0000000..78a2a68 --- /dev/null +++ b/tdbsumstat/cli/export/snp.py @@ -0,0 +1,72 @@ +"""SNP-based export from TileDB.""" +import numpy as np +import pandas as pd +import polars as pl +import tiledb + + +def export_by_snp( + tiledb_export: tiledb.Array, + snp: str, + attr: str, + type_sumstat: str, + out: str, +) -> None: + """Query TileDB by a list of SNPs and export results to CSV. + + Parameters + ---------- + tiledb_export : tiledb.Array + Open TileDB array in read mode. + snp : str + Path to a CSV file with columns CHR, POS, TRAIT (and optionally GENE for QTL). + attr : str + Comma-separated list of attributes to export. + type_sumstat : str + Type of summary statistics: "gwas" or "qtl". + out : str + Output file prefix. + """ + snp_list = pd.read_csv(snp, dtype={"CHR": int, "POS": np.uint32, "TRAIT": str}) + + if type_sumstat == "gwas": + trait_list = snp_list["TRAIT"].unique().tolist() + else: + snp_list[["CELL", "GENE"]] = snp_list["TRAIT"].str.split(":", n=2, expand=True) + trait_list = snp_list["CELL"].unique().tolist() + + for trait in trait_list: + if type_sumstat == "gwas": + chrom_list = snp_list[snp_list["TRAIT"] == trait]["CHR"].unique().tolist() + else: + chrom_list = snp_list[snp_list["CELL"] == trait]["CHR"].unique().tolist() + + for chrom in chrom_list: + if type_sumstat == "gwas": + snp_list_refined = ( + snp_list[(snp_list["CHR"] == chrom) & (snp_list["TRAIT"] == trait)]["POS"] + .unique() + .tolist() + ) + tiledb_query = tiledb_export.query( + attrs=attr.split(","), + return_arrow=True, + ).df[chrom, trait, :] + tiledb_query_pl = pl.from_arrow(tiledb_query) + tiledb_query_pd = tiledb_query_pl.filter(pl.col("POS").is_in(snp_list_refined)).to_pandas() + else: + snp_list_refined = ( + snp_list[(snp_list["CHR"] == chrom) & (snp_list["CELL"] == trait)]["POS"] + .unique() + .tolist() + ) + gene_list = ( + snp_list[(snp_list["CHR"] == chrom) & (snp_list["CELL"] == trait)]["GENE"] + .unique() + .tolist() + ) + tiledb_query = tiledb_export.query(attrs=attr.split(",")).df[chrom, trait, gene_list, :] + tiledb_query_pl = pl.from_arrow(tiledb_query) + tiledb_query_pd = tiledb_query_pl.filter(pl.col("POS").is_in(snp_list_refined)).to_pandas() + + tiledb_query_pd.to_csv(f"{out}_{trait}_{chrom}.csv", mode="a", index=False) diff --git a/tdbsumstat/cli/export/traits.py b/tdbsumstat/cli/export/traits.py new file mode 100644 index 0000000..cefbb7a --- /dev/null +++ b/tdbsumstat/cli/export/traits.py @@ -0,0 +1,54 @@ +"""Trait-based bulk export from TileDB.""" +import pandas as pd +import tiledb + + +def export_by_traits( + uri_path: str, + trait_list: str, + attr: str, + type_sumstat: str, + out: str, + batch_name: str, +) -> None: + """Export all summary statistics for a list of traits/cells-genes to CSV. + + The function streams data in chunks to handle large datasets efficiently. + + Parameters + ---------- + uri_path : str + Path to the TileDB array. + trait_list : str + Path to a CSV file with a TRAIT column (and optionally CELL/GENE columns for QTL). + attr : str + Comma-separated list of attributes to export. + type_sumstat : str + Type of summary statistics: "gwas" or "qtl". + out : str + Output file prefix. + batch_name : str + Batch identifier appended to the output file name. + """ + trait_list_pd = pd.read_csv(trait_list) + + with tiledb.open(uri_path, mode="r") as array: + if type_sumstat == "gwas": + trait_list_np = trait_list_pd["TRAIT"].to_list() + tiledb_iterator = array.query( + return_incomplete=True, + attrs=attr.split(","), + ).df[:, trait_list_np, :] + else: + trait_list_pd[["cell", "gene"]] = trait_list_pd["TRAIT"].str.split(":", expand=True) + cells = trait_list_pd["cell"].to_list() + gene = trait_list_pd["gene"].to_list() + tiledb_iterator = array.query( + return_incomplete=True, + attrs=attr.split(","), + ).df[:, cells, gene, :] + + for chunk in tiledb_iterator: + chunk.to_csv(f"{out}_{batch_name}.csv", mode="a", index=False, header=False) + + print(f"Saved filtered summary statistics in {out}") diff --git a/tdbsumstat/utils/create_metadata.py b/tdbsumstat/utils/create_metadata.py index 8c93977..4433b9e 100755 --- a/tdbsumstat/utils/create_metadata.py +++ b/tdbsumstat/utils/create_metadata.py @@ -1,69 +1,2 @@ -import json -from collections import defaultdict -input_path = "/lustre/scratch124/humgen/projects_v2/cardinal_analysis/analysis/core_dataset/freeze3/tiledbs/Tensor/TileDB_f3_UKB_cis_t2_metadata.json" -output_path = input_path.replace(".json", "_fixed.json") - -objs = [] -with open(input_path) as f: - text = f.read() - -decoder = json.JSONDecoder() -idx = 0 -while idx < len(text): - obj, idx = decoder.raw_decode(text, idx) - objs.append(obj) - -with open(output_path, "w") as f: - json.dump(objs, f, indent=2) - -print(f"✅ Salvaged {len(objs)} metadata records into {output_path}") - - -fixed_path = "/lustre/scratch124/humgen/projects_v2/cardinal_analysis/analysis/core_dataset/freeze3/tiledbs/Tensor/TileDB_f3_UKB_cis_t2_metadata_fixed.json" -output_path = fixed_path.replace("_fixed.json", "_merged.json") - -with open(fixed_path) as f: - metadata_records = json.load(f) - -# Create a merged dictionary -merged_metadata = { - "file_path": [], - "trait": [], - "celltype": [] -} - -# For storing per-cell/chromosome info if present -cell_chrom_data = defaultdict(lambda: defaultdict(list)) - -for record in metadata_records: - # Merge file paths - if "file_path" in record and record["file_path"] not in merged_metadata["file_path"]: - merged_metadata["file_path"].append(record["file_path"]) - # Merge traits - if len(record["trait"]) > 0: - for t in record["trait"]: - if t not in merged_metadata["trait"]: - merged_metadata["trait"].append(t) - # Merge celltypes and their per-chromosome data - if len(record["celltype"]) > 0: - for cell in record["celltype"]: - if cell not in merged_metadata["celltype"]: - merged_metadata["celltype"].append(cell) - # Merge chromosome-level info if exists - if cell in record: - for chrom, genes in record[cell].items(): - cell_chrom_data[cell][chrom] = {} - #existing_genes = {g for g in cell_chrom_data[cell][chrom]} - for g in genes: - #if g not in existing_genes: - cell_chrom_data[cell][chrom][g] = genes[g] - -# Add per-cell/chromosome info -for cell, chrom_dict in cell_chrom_data.items(): - merged_metadata[cell] = chrom_dict - -# Save as a single JSON object -with open(output_path, "w") as f: - json.dump(merged_metadata, f, indent=2) - -print(f"✅ Merged metadata saved to {output_path}") +# This standalone script has been moved to the top-level scripts/ directory. +# See scripts/create_metadata.py for the current version. diff --git a/tdbsumstat/utils/fix_json.py b/tdbsumstat/utils/fix_json.py index ef83c6c..45b450f 100644 --- a/tdbsumstat/utils/fix_json.py +++ b/tdbsumstat/utils/fix_json.py @@ -1,55 +1,2 @@ -import json -import re -from collections import defaultdict - -# === STEP 1: Read the entire string from file === -with open("/project/cardinal/QTLs/freeze3/TileDBs/TileDB_metanalyses_gh_celltype2_freeze3_metadata.json", "r", encoding="utf-8") as f: - data_str = f.read() - -# === STEP 2: Split based on the start of each JSON fragment === -# We capture the first '{' before each file_path -fragments = re.findall(r'(\{[^{]*"file_path":.*?)(?=\{[^{]*"file_path":|\Z)', data_str, flags=re.S) - -print(f"🧩 Found {len(fragments)} fragments") - -# === STEP 3: Parse each fragment === -parsed_fragments = [] -for i, frag in enumerate(fragments): - try: - parsed_fragments.append(json.loads(frag)) - except json.JSONDecodeError as e: - print(f"⚠️ Fragment {i} failed to parse at char {e.pos}: {e}") - print(frag[:300] + "...") - # optional: skip or raise - continue - -# === STEP 4: Merge fragments as before === -merged = { - "trait": [], - "celltype": [], -} -celltype_data = defaultdict(lambda: defaultdict(dict)) - -for frag in parsed_fragments: - if "trait" in frag and isinstance(frag["trait"], list): - merged["trait"].extend(frag["trait"]) - if "celltype" in frag and isinstance(frag["celltype"], list): - merged["celltype"].extend(frag["celltype"]) - for key, val in frag.items(): - if key in ("trait", "celltype", "CELL", "file_path"): - continue - for num, genes in val.items(): - for gene_id, gene_data in genes.items(): - celltype_data[key][num][gene_id] = gene_data - -for celltype, numbers in celltype_data.items(): - merged[celltype] = numbers - -merged["trait"] = list(dict.fromkeys(merged["trait"])) -merged["celltype"] = list(dict.fromkeys(merged["celltype"])) - -# === STEP 5: Save merged JSON === -with open("metadata_merged.json", "w", encoding="utf-8") as out_f: - json.dump(merged, out_f, indent=2) - -print("✅ Merged JSON written to metadata_merged.json") +# This standalone script has been moved to the top-level scripts/ directory. +# See scripts/fix_json.py for the current version. diff --git a/tdbsumstat/utils/generate_table_cell_sumstat.py b/tdbsumstat/utils/generate_table_cell_sumstat.py index 43a4409..8104c4f 100755 --- a/tdbsumstat/utils/generate_table_cell_sumstat.py +++ b/tdbsumstat/utils/generate_table_cell_sumstat.py @@ -1,22 +1,2 @@ -import pandas as pd -import tiledb -rows = [] -tdb = tiledb.open('/lustre/scratch124/humgen/projects_v2/cardinal_analysis/analysis/core_dataset/freeze3/tiledbs/Tensor/TileDB_f3_UKB_cis_t2', 'w') -# Loop over all celltypes in metadata -for celltype in merged_metadata["celltype"]: - if celltype in merged_metadata: # make sure the key exists in dict - groups = merged_metadata[celltype] - for group, values in groups.items(): # e.g. group "11" - for gene_id in values: - rows.append({ - "CHR": group, - "CELL": celltype, - "GENE": gene_id, - "N": values[gene_id]["N"], - "ACAT": values[gene_id]["ACAT"], - "PHENOVAR": values[gene_id]["PHENO_VAR"], - }) -df = pd.DataFrame(rows) - -tdb.meta["metadata"] = json.dumps(merged_metadata) -df.to_csv("/lustre/scratch124/humgen/projects_v2/cardinal_analysis/analysis/core_dataset/freeze3/tiledbs/Tensor/Bangladeshi/metadata_gh_bangladeshi_f3_celltype1_acat.csv",index = False) \ No newline at end of file +# This standalone script has been moved to the top-level scripts/ directory. +# See scripts/generate_table_cell_sumstat.py for the current version. diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..6b8109b --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,88 @@ +"""Pytest fixtures shared across the test suite.""" +import os +import tempfile + +import numpy as np +import pandas as pd +import polars as pl +import pytest +import tiledb + +EXAMPLE_DATA_DIR = os.path.join(os.path.dirname(__file__), "..", "example_data") + + +@pytest.fixture(scope="session") +def example_data_dir(): + """Return the path to the example_data directory.""" + return os.path.abspath(EXAMPLE_DATA_DIR) + + +@pytest.fixture(scope="session") +def mapping_file_sc(example_data_dir): + """Return the path to the QTL mapping file.""" + return os.path.join(example_data_dir, "mapping_file_test.csv") + + +@pytest.fixture(scope="session") +def qtl_tiledb(tmp_path_factory, example_data_dir, mapping_file_sc): + """Create a QTL TileDB from example data and return its path. + + This fixture runs once per test session and reuses the same TileDB. + """ + from tdbsumstat.utils.harmonize_ingest import Harmonize + + uri = str(tmp_path_factory.mktemp("tiledb") / "test_qtl_tiledb") + + h = Harmonize( + mapping_file=mapping_file_sc, + uri=uri, + type_sumstat="qtl", + pvar_file=None, + type_trait="quant", + mac=None, + maf=None, + permuted=False, + ) + h.create_tiledb() + h.create_mapping() + + # Ingest both example files + data_files = [ + os.path.join(example_data_dir, "dummy_out_ENSG0000010000.tsv.gz"), + os.path.join(example_data_dir, "dummy_out_ENSG0000010001.tsv.gz"), + ] + + for filepath in data_files: + chunk_pl = pl.read_csv(filepath, separator="\t", low_memory=True, null_values="NA") + # Extract gene from filename: dummy_out_ENSG0000010000.tsv.gz -> ENSG0000010000 + gene = os.path.basename(filepath).replace("dummy_out_", "").split(".")[0] + h.harmonize(sumstat=chunk_pl, cell="T_gd", gene=gene, n=4000, pheno_var=1.5) + h.ingest_data(file_path=filepath) + h.create_metadata(file_path=filepath) + + h.merge_metadata_files() + return uri + + +@pytest.fixture(scope="session") +def qtl_metadata_df(qtl_tiledb): + """Return the metadata DataFrame for the QTL TileDB.""" + import json + + with tiledb.open(qtl_tiledb, mode="r") as A: + metadata = json.loads(A.meta["merged_metadata"]) + + rows = [] + for cell in metadata["CELL"]: + for chrom, genes in metadata[cell].items(): + for gene, stats in genes.items(): + rows.append({"CELL": cell, "CHR": chrom, "GENE": gene, **stats}) + df_meta = pl.DataFrame(rows) + df_meta = df_meta.with_columns(pl.col("CHR").cast(pl.UInt16)) + return df_meta + + +@pytest.fixture() +def tmp_out(tmp_path): + """Return a temporary output prefix for test files.""" + return str(tmp_path / "out") diff --git a/tests/test_export.py b/tests/test_export.py new file mode 100644 index 0000000..d92cea5 --- /dev/null +++ b/tests/test_export.py @@ -0,0 +1,147 @@ +"""Tests for the export handler functions (snp, regions, locusbreaker, metadata, traits).""" +import os + +import pandas as pd +import polars as pl +import pytest +import tiledb + +from tdbsumstat.cli.export.regions import export_by_regions +from tdbsumstat.cli.export.snp import export_by_snp + +EXAMPLE_DATA_DIR = os.path.join(os.path.dirname(__file__), "..", "example_data") + + +class TestExportByRegions: + def test_qtl_regions_output(self, qtl_tiledb, tmp_out): + """Export by regions should create a non-empty CSV for known regions.""" + region_file = os.path.join(EXAMPLE_DATA_DIR, "region_list_sc.csv") + with tiledb.open(qtl_tiledb, mode="r") as tdb: + export_by_regions( + tiledb_export=tdb, + table_regions=region_file, + attr="P,SNPID,EAF,BETA,SE", + type_sumstat="qtl", + out=tmp_out + "_regions.csv", + ) + + output_file = tmp_out + "_regions.csv" + if os.path.exists(output_file): + df = pd.read_csv(output_file) + assert len(df) > 0 + assert "P" in df.columns + + def test_empty_region_produces_no_file(self, qtl_tiledb, tmp_out, tmp_path): + """Querying a region with no matching gene should not create an output file.""" + empty_region_file = str(tmp_path / "empty_region.csv") + # Use valid CHR/cell but a gene that doesn't exist in example data + pd.DataFrame({ + "CHR": [20], + "START": [1], + "END": [100], + "TRAIT": ["T_gd:ENSG9999999999"], + }).to_csv(empty_region_file, index=False) + + out_file = tmp_out + "_empty.csv" + with tiledb.open(qtl_tiledb, mode="r") as tdb: + export_by_regions( + tiledb_export=tdb, + table_regions=empty_region_file, + attr="P,SNPID,EAF,BETA,SE", + type_sumstat="qtl", + out=out_file, + ) + # Should not have created a file (no data matched) + assert not os.path.exists(out_file) + + +class TestExportBySnp: + def test_qtl_snp_output(self, qtl_tiledb, tmp_out): + """Export by SNP list should create a CSV with matching rows.""" + snp_file = os.path.join(EXAMPLE_DATA_DIR, "snp_list_sc.csv") + with tiledb.open(qtl_tiledb, mode="r") as tdb: + export_by_snp( + tiledb_export=tdb, + snp=snp_file, + attr="P,SNPID,EAF,BETA,SE", + type_sumstat="qtl", + out=tmp_out, + ) + # The output file(s) are named {out}_{trait}_{chrom}.csv + output_files = [ + f for f in os.listdir(os.path.dirname(tmp_out) or ".") + if f.endswith(".csv") + ] + assert len(output_files) > 0 + + +class TestLocusbreakerExport: + def test_locusbreaker_import(self): + """Locusbreaker module should be importable without errors.""" + from tdbsumstat.cli.export.locusbreaker import export_with_locusbreaker # noqa: F401 + + +class TestMetadataExport: + def test_export_metadata_creates_file(self, qtl_tiledb, tmp_out): + """export_metadata should create a *_meta.csv file.""" + from tdbsumstat.cli.export.metadata import export_metadata + + export_metadata(qtl_tiledb, "qtl", tmp_out) + assert os.path.exists(tmp_out + "_meta.csv") + df = pd.read_csv(tmp_out + "_meta.csv") + assert len(df) > 0 + + def test_recompute_metadata_import(self): + """recompute_metadata should be importable without errors.""" + from tdbsumstat.cli.export.metadata import recompute_metadata # noqa: F401 + + +class TestExportByTraits: + def test_export_by_traits_import(self): + """export_by_traits should be importable without errors.""" + from tdbsumstat.cli.export.traits import export_by_traits # noqa: F401 + + def test_export_by_traits_qtl(self, qtl_tiledb, tmp_out): + """Export by traits should run without errors.""" + from tdbsumstat.cli.export.traits import export_by_traits + + trait_file = str(os.path.join(os.path.dirname(tmp_out), "trait_list.csv")) + pd.DataFrame({"TRAIT": ["T_gd:ENSG0000010000"]}).to_csv(trait_file, index=False) + + export_by_traits( + uri_path=qtl_tiledb, + trait_list=trait_file, + attr="P,SNPID,EAF,BETA,SE", + type_sumstat="qtl", + out=tmp_out, + batch_name="test", + ) + # The function completes without error; output may be empty for small example data + + +class TestModuleImports: + """Smoke tests: ensure all new modules can be imported cleanly.""" + + def test_import_helpers(self): + from tdbsumstat.cli.export.helpers import open_tiledb_and_load_metadata # noqa: F401 + + def test_import_snp(self): + from tdbsumstat.cli.export.snp import export_by_snp # noqa: F401 + + def test_import_regions(self): + from tdbsumstat.cli.export.regions import export_by_regions # noqa: F401 + + def test_import_locusbreaker(self): + from tdbsumstat.cli.export.locusbreaker import export_with_locusbreaker # noqa: F401 + + def test_import_metadata(self): + from tdbsumstat.cli.export.metadata import export_metadata, recompute_metadata # noqa: F401 + + def test_import_traits(self): + from tdbsumstat.cli.export.traits import export_by_traits # noqa: F401 + + def test_import_command(self): + from tdbsumstat.cli.export.command import export # noqa: F401 + + def test_import_export_package(self): + from tdbsumstat.cli.export import export # noqa: F401 diff --git a/tests/test_harmonize.py b/tests/test_harmonize.py new file mode 100644 index 0000000..86c7141 --- /dev/null +++ b/tests/test_harmonize.py @@ -0,0 +1,156 @@ +"""Tests for the Harmonize class (harmonize_ingest.py).""" +import os + +import numpy as np +import polars as pl +import pytest +import tiledb + +from tdbsumstat.utils.harmonize_ingest import Harmonize, HarmonizationError + + +EXAMPLE_DATA_DIR = os.path.join(os.path.dirname(__file__), "..", "example_data") + + +@pytest.fixture() +def mapping_file(): + return os.path.join(EXAMPLE_DATA_DIR, "mapping_file_test.csv") + + +@pytest.fixture() +def sample_qtl_df(): + """Minimal QTL polars DataFrame mimicking the example data columns.""" + return pl.DataFrame({ + "Chr": [20, 20, 20], + "Gene": ["ENSG0000010000", "ENSG0000010000", "ENSG0000010000"], + "cell.type": ["T_gd", "T_gd", "T_gd"], + "pos": [100, 200, 300], + "a0": ["A", "C", "G"], + "a1": ["T", "G", "A"], + "p": [0.01, 0.5, 0.001], + "N": [500, 500, 500], + "beta": [0.1, -0.2, 0.3], + "se": [0.05, 0.08, 0.04], + }) + + +class TestCreateMapping: + def test_valid_mapping(self, mapping_file): + h = Harmonize(mapping_file, "dummy", "qtl", None, "quant", None, None, False) + h.create_mapping() + assert "CHR" in h.mapping_types.values() + assert "BETA" in h.mapping_types.values() + assert "SE" in h.mapping_types.values() + + def test_missing_beta_se_raises(self, tmp_path): + bad_mapping = tmp_path / "bad_mapping.csv" + bad_mapping.write_text("Chr,CHR\npos,POS\n") + h = Harmonize(str(bad_mapping), "dummy", "qtl", None, "quant", None, None, False) + with pytest.raises(HarmonizationError, match="BETA and SE"): + h.create_mapping() + + def test_empty_mapping_raises(self, tmp_path): + empty_file = tmp_path / "empty.csv" + empty_file.write_text("") + h = Harmonize(str(empty_file), "dummy", "qtl", None, "quant", None, None, False) + with pytest.raises(HarmonizationError, match="empty or not formatted"): + h.create_mapping() + + +class TestCreateTileDB: + def test_creates_qtl_tiledb(self, mapping_file, tmp_path): + uri = str(tmp_path / "test_tiledb") + h = Harmonize(mapping_file, uri, "qtl", None, "quant", None, None, False) + h.create_tiledb() + assert tiledb.array_exists(uri) + schema = tiledb.ArraySchema.load(uri) + dim_names = [schema.domain.dim(i).name for i in range(schema.domain.ndim)] + assert "CHR" in dim_names + assert "CELL" in dim_names + assert "GENE" in dim_names + assert "POS" in dim_names + + def test_creates_gwas_tiledb(self, mapping_file, tmp_path): + uri = str(tmp_path / "test_gwas") + h = Harmonize(mapping_file, uri, "gwas", None, "quant", None, None, False) + h.create_tiledb() + assert tiledb.array_exists(uri) + schema = tiledb.ArraySchema.load(uri) + dim_names = [schema.domain.dim(i).name for i in range(schema.domain.ndim)] + assert "TRAIT" in dim_names + assert "CHR" in dim_names + assert "POS" in dim_names + + +class TestHarmonize: + def test_harmonize_qtl_adds_cell_gene(self, mapping_file, sample_qtl_df): + h = Harmonize(mapping_file, "dummy", "qtl", None, "quant", None, None, False) + h.create_mapping() + h.harmonize(sumstat=sample_qtl_df, cell="T_gd", gene="ENSG0000010000", n=500, pheno_var=1.5) + assert "CELL" in h.chunk_pl.columns + assert "GENE" in h.chunk_pl.columns + assert "SNPID" in h.chunk_pl.columns + + def test_harmonize_snpid_format(self, mapping_file, sample_qtl_df): + h = Harmonize(mapping_file, "dummy", "qtl", None, "quant", None, None, False) + h.create_mapping() + h.harmonize(sumstat=sample_qtl_df, cell="T_gd", gene="ENSG0000010000", n=500, pheno_var=1.5) + # All SNPIDs should start with 'chr' + snpids = h.chunk_pl["SNPID"].to_list() + assert all(s.startswith("chr") for s in snpids) + + def test_harmonize_raises_without_n(self, mapping_file, sample_qtl_df): + df_no_n = sample_qtl_df.drop("N") + h = Harmonize(mapping_file, "dummy", "qtl", None, "quant", None, None, False) + h.create_mapping() + with pytest.raises(HarmonizationError): + h.harmonize(sumstat=df_no_n, cell="T_gd", gene="ENSG0000010000", pheno_var=1.5) + + def test_harmonize_maf_filter(self, mapping_file, sample_qtl_df): + """MAF filter should remove rows with EAF outside the range.""" + # Add EAF column: use a0/a1 encoded EAF from original data, but override + df_with_eaf = sample_qtl_df.with_columns(pl.lit(0.01).alias("EAF_explicit")) + h = Harmonize(mapping_file, "dummy", "qtl", None, "quant", None, maf=0.05, permuted=False) + h.create_mapping() + # With maf=0.05 and EAF derived from alleles, some rows may be filtered + # Just make sure the call doesn't crash + h.harmonize(sumstat=sample_qtl_df, cell="T_gd", gene="ENSG0000010000", n=500, pheno_var=1.5) + + def test_binary_trait_adds_n_columns(self, mapping_file, sample_qtl_df): + h = Harmonize(mapping_file, "dummy", "gwas", None, "binary", None, None, False) + h.create_mapping() + gwas_df = sample_qtl_df.rename({"cell.type": "TRAIT_col", "Gene": "TRAIT"}).drop("Gene", strict=False) + # Use a simplified gwas-like df + gwas_df = pl.DataFrame({ + "Chr": [1, 1], + "pos": [100, 200], + "a0": ["A", "C"], + "a1": ["T", "G"], + "p": [0.01, 0.5], + "beta": [0.1, -0.2], + "se": [0.05, 0.08], + "N": [500, 500], + "Gene": ["T1", "T1"], + "cell.type": ["gwas", "gwas"], + }) + mapping_gwas = mapping_file # mapping is for QTL, adapt + h2 = Harmonize(mapping_gwas, "dummy", "gwas", None, "binary", None, None, False) + h2.create_mapping() + h2.harmonize( + sumstat=gwas_df, + trait="T1", + n_cases=200, + n_controls=300, + pheno_var=1.0, + ) + assert "N_CASES" in h2.chunk_pl.columns + assert "N_CONTROLS" in h2.chunk_pl.columns + + +class TestIngestData: + def test_ingest_creates_data(self, qtl_tiledb): + """The session-scoped qtl_tiledb fixture should have rows.""" + with tiledb.open(qtl_tiledb, mode="r") as A: + df = A.query(dims=["CHR", "CELL", "GENE", "POS"]).df[:] + assert len(df) > 0 + assert "CELL" in df.columns diff --git a/tests/test_utils.py b/tests/test_utils.py new file mode 100644 index 0000000..a3c0359 --- /dev/null +++ b/tests/test_utils.py @@ -0,0 +1,121 @@ +"""Tests for utility functions (acat_optimized, compute_pheno_variance, z_to_p_via_chi2).""" +import math + +import numpy as np +import polars as pl +import pytest + +from tdbsumstat.utils import acat_optimized, compute_pheno_variance, z_to_p_via_chi2 + + +class TestAcatOptimized: + def test_uniform_pvalues(self): + """ACAT of many uniform p-values should be close to 0.5.""" + pvals = pl.Series([0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]) + result = acat_optimized(pvals) + assert 0.0 <= result <= 1.0 + + def test_very_small_pvalue_list(self): + """A single very small p-value should produce a small ACAT.""" + pvals = pl.Series([1e-20]) + result = acat_optimized(pvals) + assert result < 1e-10 + + def test_all_ones(self): + """p-values of 1 should produce ACAT close to 1.""" + pvals = pl.Series([1.0, 1.0, 1.0]) + result = acat_optimized(pvals) + assert result > 0.9 + + def test_accepts_list(self): + """acat_optimized should accept plain Python lists.""" + pvals = [0.05, 0.1, 0.2] + result = acat_optimized(pvals) + assert 0.0 <= result <= 1.0 + + def test_invalid_type_raises(self): + """Non-Series / non-list input should raise TypeError.""" + with pytest.raises(TypeError): + acat_optimized(np.array([0.05, 0.1])) + + def test_out_of_range_raises(self): + """p-values outside [0, 1] should raise ValueError.""" + pvals = pl.Series([0.5, 1.5]) + with pytest.raises(ValueError): + acat_optimized(pvals) + + def test_nan_values_ignored(self): + """NaN p-values should be silently ignored.""" + pvals = pl.Series([0.05, float("nan"), 0.1]) + result = acat_optimized(pvals) + assert 0.0 <= result <= 1.0 + + def test_all_nan_returns_nan(self): + """All-NaN input should return NaN.""" + pvals = pl.Series([float("nan"), float("nan")]) + result = acat_optimized(pvals) + assert math.isnan(result) + + def test_small_pvalue_approximation(self): + """Values below 1e-15 should use 1/(pi*p) approximation without crashing.""" + pvals = pl.Series([1e-300, 0.05]) + result = acat_optimized(pvals) + assert result < 0.05 # very small p drives ACAT down + + +class TestComputePhenoVariance: + def _make_quant_df(self, n=500, eaf=0.3, se=0.1, beta=0.05): + return pl.DataFrame({ + "N": [n] * 10, + "EAF": [eaf] * 10, + "SE": [se] * 10, + "BETA": [beta] * 10, + }) + + def _make_binary_df(self, n_cases=200, n_controls=300, eaf=0.3, se=0.1, beta=0.05): + return pl.DataFrame({ + "N_CASES": [float(n_cases)] * 10, + "N_CONTROLS": [float(n_controls)] * 10, + "EAF": [eaf] * 10, + "SE": [se] * 10, + "BETA": [beta] * 10, + }) + + def test_quant_returns_string(self): + df = self._make_quant_df() + result = compute_pheno_variance(df, "quant") + # The function returns a string representation of a float + assert isinstance(result, str) + float(result) # should not raise + + def test_binary_adds_n_column(self): + df = self._make_binary_df() + result = compute_pheno_variance(df, "binary") + assert isinstance(result, str) + float(result) + + def test_quant_positive_variance(self): + df = self._make_quant_df(n=1000, eaf=0.4, se=0.05) + result = float(compute_pheno_variance(df, "quant")) + assert result > 0 + + +class TestZToPViaChi2: + def test_zero_z_gives_one(self): + """z=0 should give p-value of 1.""" + p = z_to_p_via_chi2(0) + assert abs(p - 1.0) < 1e-10 + + def test_large_z_gives_small_p(self): + """Large z-score should give very small p-value.""" + p = z_to_p_via_chi2(10) + assert p < 1e-20 + + def test_z_196_gives_approx_005(self): + """z ≈ 1.96 should give p ≈ 0.05.""" + p = z_to_p_via_chi2(1.96) + assert abs(p - 0.05) < 0.01 + + def test_symmetric(self): + """Positive and negative z should give the same p-value.""" + assert abs(z_to_p_via_chi2(2.5) - z_to_p_via_chi2(-2.5)) < 1e-15 From e461931efcc5c19c2a2ace2ae41e1a6ff00b151f Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 5 Mar 2026 19:32:37 +0000 Subject: [PATCH 3/6] Refactor: split harmonize_ingest.py into focused ingest sub-modules + 32 tests + README update Co-authored-by: bruno-ariano <26384813+bruno-ariano@users.noreply.github.com> --- README.md | 116 +++++ tdbsumstat/cli/ingestion.py | 2 +- tdbsumstat/utils/harmonize_ingest.py | 694 +-------------------------- tdbsumstat/utils/ingest/__init__.py | 92 ++++ tdbsumstat/utils/ingest/errors.py | 6 + tdbsumstat/utils/ingest/harmonize.py | 272 +++++++++++ tdbsumstat/utils/ingest/mapping.py | 41 ++ tdbsumstat/utils/ingest/metadata.py | 284 +++++++++++ tdbsumstat/utils/ingest/qc.py | 69 +++ tdbsumstat/utils/ingest/schema.py | 80 +++ tdbsumstat/utils/ingest/writer.py | 61 +++ tests/test_ingest.py | 391 +++++++++++++++ 12 files changed, 1433 insertions(+), 675 deletions(-) create mode 100644 tdbsumstat/utils/ingest/__init__.py create mode 100644 tdbsumstat/utils/ingest/errors.py create mode 100644 tdbsumstat/utils/ingest/harmonize.py create mode 100644 tdbsumstat/utils/ingest/mapping.py create mode 100644 tdbsumstat/utils/ingest/metadata.py create mode 100644 tdbsumstat/utils/ingest/qc.py create mode 100644 tdbsumstat/utils/ingest/schema.py create mode 100644 tdbsumstat/utils/ingest/writer.py create mode 100644 tests/test_ingest.py diff --git a/README.md b/README.md index b983916..d1ed1e6 100755 --- a/README.md +++ b/README.md @@ -40,5 +40,121 @@ If you are running the pipeline with nextflow change the path where the conda en #### To check how to use the Nextflow pipeline for extracting and ingesting data please refer here [HERE](https://github.com/HTGenomeAnalysisUnit/TileDB-sumstat/blob/nextflow_branch/docs/README.md) +--- + +### Code structure + +The Python package lives under `tdbsumstat/` and is organised as follows: + +``` +tdbsumstat/ +├── main.py # CLI entry-point (registers ingest + export commands) +├── cli/ +│ ├── ingestion.py # `tdbsumstat ingest` CLI command +│ └── export/ # `tdbsumstat export` CLI command (package) +│ ├── __init__.py # re-exports the `export` command +│ ├── command.py # CLI decorator + routing to handlers +│ ├── helpers.py # shared TileDB open/metadata helper +│ ├── snp.py # export by SNP list +│ ├── regions.py # export by genomic regions +│ ├── locusbreaker.py # locusbreaker wrapper +│ ├── metadata.py # metadata export / recompute +│ └── traits.py # bulk trait export +└── utils/ + ├── __init__.py # acat_optimized, compute_pheno_variance, z_to_p_via_chi2 + ├── harmonize_ingest.py # backward-compat shim → re-exports from utils/ingest/ + ├── ingest/ # ingestion pipeline (package) + │ ├── __init__.py # Harmonize class + HarmonizationError + │ ├── errors.py # HarmonizationError exception + │ ├── schema.py # SchemaMixin – TileDB array creation + │ ├── mapping.py # MappingMixin – column-mapping CSV parsing + │ ├── harmonize.py # HarmonizeMixin – data normalisation + │ ├── qc.py # QCMixin – optional gwaslab QC + │ ├── writer.py # WriterMixin – TileDB data writer + │ └── metadata.py # MetadataMixin – metadata management + ├── locusbreaker.py # pandas-based locusbreaker (legacy) + ├── locusbreaker_plpl.py # Polars-based locusbreaker (used by export) + └── update_metadata.py # standalone metadata update utility +scripts/ + ├── create_metadata.py # one-off metadata creation helper + ├── fix_json.py # one-off JSON repair helper + └── generate_table_cell_sumstat.py # one-off table generation helper +``` + +#### Ingestion pipeline in detail + +The `Harmonize` class (in `tdbsumstat/utils/ingest/`) is composed from focused mixin classes: + +| Module | Mixin | Responsibility | +|--------|-------|----------------| +| `schema.py` | `SchemaMixin` | Create the TileDB sparse array schema | +| `mapping.py` | `MappingMixin` | Parse the column-mapping CSV | +| `harmonize.py` | `HarmonizeMixin` | Rename columns, handle alleles, compute p-values | +| `qc.py` | `QCMixin` | Optional gwaslab-based QC checks | +| `writer.py` | `WriterMixin` | Deduplicate and append data to TileDB | +| `metadata.py` | `MetadataMixin` | Create, merge and export metadata JSON/CSV | + +A typical ingestion workflow (Python API): + +```python +from tdbsumstat.utils.ingest import Harmonize +import polars as pl + +h = Harmonize( + mapping_file="mapping.csv", + uri="my_tiledb", + type_sumstat="qtl", # or "gwas" + pvar_file=None, + type_trait="quant", # or "binary" + mac=None, + maf=None, + permuted=False, +) + +# One-time setup +h.create_tiledb() +h.create_mapping() + +# Per-file loop +for filepath, cell, gene in file_list: + sumstat = pl.read_csv(filepath, separator="\t", null_values="NA") + h.harmonize(sumstat=sumstat, cell=cell, gene=gene, n=n, pheno_var=pheno_var) + h.ingest_data(file_path=filepath) + h.create_metadata(file_path=filepath) + +# Finalise metadata +h.merge_metadata_files() +``` + +--- + +### Running tests + +Tests live in the `tests/` directory and use [pytest](https://pytest.org). + +```bash +# Install dev dependencies +pip install pytest pytest-cov + +# Run the full test suite +python -m pytest tests/ -v + +# Run only ingestion tests +python -m pytest tests/test_ingest.py -v + +# Run only export tests +python -m pytest tests/test_export.py -v + +# Run with coverage report +python -m pytest tests/ --cov=tdbsumstat --cov-report=term-missing +``` + +Test files: +| File | What it tests | +|------|---------------| +| `tests/test_utils.py` | `acat_optimized`, `compute_pheno_variance`, `z_to_p_via_chi2` | +| `tests/test_harmonize.py` | `Harmonize` class (legacy + backward-compat) | +| `tests/test_ingest.py` | Each ingest mixin + end-to-end pipeline with example data | +| `tests/test_export.py` | All export modules + CLI command routing | diff --git a/tdbsumstat/cli/ingestion.py b/tdbsumstat/cli/ingestion.py index 0c45a6b..17cfd30 100755 --- a/tdbsumstat/cli/ingestion.py +++ b/tdbsumstat/cli/ingestion.py @@ -3,7 +3,7 @@ import os import click import cloup -from tdbsumstat.utils.harmonize_ingest import Harmonize +from tdbsumstat.utils.ingest import Harmonize import pandas as pd import polars as pl diff --git a/tdbsumstat/utils/harmonize_ingest.py b/tdbsumstat/utils/harmonize_ingest.py index ed55b5d..58891b0 100755 --- a/tdbsumstat/utils/harmonize_ingest.py +++ b/tdbsumstat/utils/harmonize_ingest.py @@ -1,674 +1,20 @@ -import logging -import json -from pathlib import Path -import os -import pandas as pd -import polars as pl -import numpy as np -from scipy import stats -import tiledb -import gwaslab as gl -from collections import defaultdict -from tdbsumstat.utils import acat_optimized, compute_pheno_variance - - -logger = logging.getLogger(__name__) -logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s") - - -class HarmonizationError(Exception): - """Custom exception for harmonization errors.""" - pass - - -class Harmonize: - def __init__(self, mapping_file: str, uri: str, type_sumstat: str, pvar_file: str, type_trait: str, mac: int, maf: float, permuted: bool): - self.mapping_file = mapping_file - self.uri = uri - self.pvar_file = pvar_file - self.type_sumstat = type_sumstat - self.type_trait = type_trait - self.mapping_types = {} - self.tiledb_types = {} - self.dimension_tiledb = [] - self.mac = mac - self.maf = maf - self.permuted = permuted - print(self.permuted) - - def create_mapping(self): - df = pd.read_csv(self.mapping_file, header=None, names=["key", "value"]) - if df.empty: - raise HarmonizationError("Mapping file is empty or not formatted correctly.") - self.mapping_types = dict(zip(df["key"], df["value"])) - #check that "BETA", "SE" are in the vlaues of the mapping_types - if not all(col in self.mapping_types.values() for col in ["BETA", "SE"]): - raise HarmonizationError("Mapping file must contain BETA and SE columns.") - # Check if CHR and POS or SNP are present - if not all(col for col in ["CHR", "POS"] if col in self.mapping_types.values()): - if "SNPID" not in self.mapping_types.values(): - raise HarmonizationError("Mapping file must contain either CHR, and POS or SNPID columns.") - - def create_tiledb(self): - """Create the mapping and dtype definitions.""" - pos_domain = (1, 300000000) # Example range for genomic positions - chr_domain = (1, 24) # Example range for genomic positions - attrs=[ - tiledb.Attr(name="SNPID", dtype="ascii", filters=tiledb.FilterList([tiledb.ZstdFilter(level=5)])), - tiledb.Attr(name="RSID", dtype="ascii", filters=tiledb.FilterList([tiledb.ZstdFilter(level=5)])), - tiledb.Attr(name="EAF", dtype=np.float32, filters=tiledb.FilterList([tiledb.ZstdFilter(level=5)])), - tiledb.Attr(name="BETA", dtype=np.float32, filters=tiledb.FilterList([tiledb.ZstdFilter(level=5)])), - tiledb.Attr(name="SE", dtype=np.float32, filters=tiledb.FilterList([tiledb.ZstdFilter(level=5)])), - tiledb.Attr(name="P", dtype=np.float64, filters=tiledb.FilterList([tiledb.ZstdFilter(level=5)])) - ] - - if self.type_sumstat == "gwas": - - self.dimension_tiledb = ["CHR", "TRAIT", "POS"] - dom = tiledb.Domain( - tiledb.Dim(name="CHR", domain = chr_domain, dtype=np.uint16, filters=tiledb.FilterList([tiledb.ZstdFilter(level=5)])), - tiledb.Dim(name="TRAIT", dtype="ascii", var=True, filters=tiledb.FilterList([tiledb.ZstdFilter(level=5)]) ), - tiledb.Dim(name="POS", domain = pos_domain, dtype=np.uint32, filters=tiledb.FilterList([tiledb.ZstdFilter(level=5)])) - ) - else: - - self.dimension_tiledb = ["CHR", "CELL", "GENE", "POS"] - dom = tiledb.Domain( - tiledb.Dim(name="CHR", domain = chr_domain, dtype=np.uint16, filters=tiledb.FilterList([tiledb.ZstdFilter(level=5)])), - tiledb.Dim(name="CELL", dtype="ascii", var=True, filters=tiledb.FilterList([tiledb.ZstdFilter(level=5)]) ), - tiledb.Dim(name="GENE",dtype="ascii", var=True, filters=tiledb.FilterList([tiledb.ZstdFilter(level=5)]) ), - tiledb.Dim(name="POS", domain = pos_domain, dtype=np.uint32, filters=tiledb.FilterList([tiledb.ZstdFilter(level=5)])) - ) - - attrs = attrs + [ - tiledb.Attr(name="DIST", dtype=np.float32, filters=tiledb.FilterList([tiledb.ZstdFilter(level=5)])) - ] - schema = tiledb.ArraySchema( - domain=dom, - attrs=attrs, - sparse=True, - allows_duplicates=False - ) - tiledb.Array.create(self.uri, schema) - - def align_alleles(self): - """Align alleles based on pvar file.""" - if not self.pvar_file: - raise HarmonizationError("pvar_file must be provided to verify alleles order") - if not Path(self.pvar_file).is_file(): - raise FileNotFoundError(f"pvar_file {self.pvar_file} does not exist") - - #The ALT must correspond to the alternative allele - pvar_df = pl.read_csv( - self.pvar_file, - separator="\t", - has_header=True, - dtypes={"CHROM": pl.Utf8, "POS": pl.Utf8, "SNPID": pl.Utf8, - "REF": pl.Utf8, "ALT": pl.Utf8}, - ) - #Here we assume the SNPID is alphabetically sortedin both pvar and summary statistics - self.chunk_pl = self.chunk_pl.join(pvar_df, on="SNPID", how="inner", suffix="_pvar") - - swap = pl.col("ALT")< pl.col("REF") - - self.chunk_pl = self.chunk_pl.with_columns([ - pl.when(swap).then(pl.col("BETA")).otherwise(-pl.col("BETA")), - pl.when(swap).then(pl.col("EAF")).otherwise(1.0-pl.col("EAF")) - ]) - - def harmonize(self, - sumstat, - trait: str = None, - cell: str = None, - gene: str = None, - pheno_var:int = None, - n: int = None, - n_controls: int = None, - n_cases: int = None): - """Load and rename columns, and ensure CHR/POS exist.""" - self.chunk_pl = sumstat.rename(self.mapping_types) - # If CHR/POS missing, extract from SNP ID - if "CHR" not in self.chunk_pl.columns or "POS" not in self.chunk_pl.columns: - self.chunk_pl = self.chunk_pl.with_columns( - pl.col("SNPID") - .str.split_exact(":", 4) - .struct.rename_fields(["CHR", "POS", "A1", "A2"]) - .alias("fields") - ).unnest("fields") - - if "SNPID" in self.chunk_pl.columns: - self.chunk_pl = self.chunk_pl.drop("SNPID") - if "EAF" not in self.chunk_pl.columns: - self.chunk_pl = self.chunk_pl.with_columns(pl.lit(0).alias("EAF")) - if "DIST" not in self.chunk_pl.columns: - self.chunk_pl = self.chunk_pl.with_columns(pl.lit(1).alias("DIST")) - - if self.maf is not None: - self.chunk_pl = self.chunk_pl.with_columns( - (pl.min_horizontal(pl.col("EAF"), 1 - pl.col("EAF"))) - .alias("MAF") - ).filter(pl.col("MAF") >= self.maf) - - if self.type_trait == "quant": - if not "N" in self.chunk_pl.columns: - if n is not None: - self.chunk_pl = self.chunk_pl.with_columns( - pl.lit(int(n)).alias("N") - ) - else: - raise HarmonizationError("N column is missing and N parameter is not provided") - if self.mac is not None: - self.chunk_pl = self.chunk_pl.with_columns( - (2 * pl.col("N") * pl.min_horizontal(pl.col("EAF"), 1 - pl.col("EAF"))) - .alias("MAC") - ).filter(pl.col("MAC") >= self.mac) - - self.chunk_pl = self.chunk_pl.with_columns( - pl.lit(pheno_var).alias("PHENO_VAR") - ) - elif self.type_trait == "binary": - if not all(sample_size in self.chunk_pl.columns for sample_size in ["N_CASES", "N_CONTROLS"]): - if not None in [n_cases, n_controls]: - self.chunk_pl = self.chunk_pl.with_columns( - pl.lit(float(n_cases)).alias("N_CASES"), - pl.lit(float(n_controls)).alias("N_CONTROLS"), - pl.lit(float(n_cases) + float(n_controls)).alias("N"), - ) - else: - raise HarmonizationError("n_cases and n_controls columns are missing and were not provided") - if self.mac is not None: - self.chunk_pl = self.chunk_pl.with_columns( - (2 * pl.col("N") * pl.min_horizontal(pl.col("EAF"), 1 - pl.col("EAF"))) - .alias("MAC") - ).filter(pl.col("MAC") >= self.mac) - else: - raise HarmonizationError("Type of trait must be either binary or quant") - - - - if "SNPID" in self.chunk_pl.columns: - self.chunk_pl = self.chunk_pl.drop("SNPID") - - - #Here we always assumbe that the SNPs are in REF=A1 and ALT=A2 - swap = pl.col("A1") < pl.col("A2") - - #Start by creating new SNPID aligned - self.chunk_pl = self.chunk_pl.with_columns([ - pl.when(swap).then(pl.col("A1")).otherwise(pl.col("A2")).alias("EA"), - # set NEA to the larger allele - pl.when(swap).then(pl.col("A2")).otherwise(pl.col("A1")).alias("NEA")] - ) - - self.chunk_pl = self.chunk_pl.with_columns( - pl.concat_str( - [ - pl.col("CHR"), - pl.col("POS").cast(pl.Utf8), # cast POS if numeric - pl.col("EA"), # lexicographically smaller - pl.col("NEA") # lexicographically larger - ], - separator=":" - ).alias("SNPID") - ) - - if self.pvar_file: - self.align_alleles() - self.chunk_pl.drop(["REF","ALT"]) - else: - # flip the sign of BETA when swapping - self.chunk_pl = self.chunk_pl.with_columns([ - pl.when(swap).then(-pl.col("BETA")).otherwise(pl.col("BETA")).alias("BETA"), - # flip EAF to 1 - EAF when swapping - pl.when(swap).then(1.0 - pl.col("EAF")).otherwise(pl.col("EAF")).alias("EAF"), - # set EA to the smaller allele - ]) - self.chunk_pl = self.chunk_pl.drop(["A1","A2"]) - - self.chunk_pl = self.chunk_pl.with_columns( - pl.concat_str( - pl.lit("chr"), - pl.col("SNPID")).alias("SNPID")) - - if self.type_sumstat=="gwas": - self.tiledb_types = { - "CHR": np.uint16, - "TRAIT": str, - "POS": np.uint32, - "SNPID": str, - "RSID": str, - "EAF": np.float32, - "BETA": np.float32, - "SE": np.float32, - "P": np.float64, - } - if "TRAIT" not in self.chunk_pl.columns: - self.chunk_pl = self.chunk_pl.with_columns( - pl.lit(trait).alias("TRAIT") - ) - else: - self.tiledb_types = { - "CHR": np.uint16, - "CELL": str, - "GENE": str, - "POS": np.uint32, - "SNPID": str, - "RSID": str, - "DIST": np.int64, - "EAF": np.float32, - "BETA": np.float32, - "SE": np.float32, - "P": np.float64, - } - if "CELL" not in self.chunk_pl.columns: - self.chunk_pl = self.chunk_pl.with_columns( - pl.lit(cell).alias("CELL") - ) - if "GENE" not in self.chunk_pl.columns: - self.chunk_pl = self.chunk_pl.with_columns( - pl.lit(gene).alias("GENE") - ) - if "RSID" not in self.chunk_pl.columns: - self.chunk_pl = self.chunk_pl.with_columns( - pl.lit("None").alias("RSID") - ) - if "LOG10P" in self.chunk_pl.columns: - self.chunk_pl = self.chunk_pl.with_columns( - (10 ** (-pl.col("LOG10P"))).alias("P") - ) - - - if self.permuted: - self.chunk_pl = self.chunk_pl.with_columns( - ( - pl.col("BETA").pow(2) / - pl.col("P").map_batches( - lambda x: pl.Series(stats.chi2.isf(x.to_numpy(), df=1)) - ) - ).sqrt().alias("SE")) - - else: - #Calculate p-value from z-score - self.chunk_pl = self.chunk_pl.drop('P') - self.chunk_pl = self.chunk_pl.with_columns( - (pl.col("BETA") / pl.col("SE")).pow(2).map_batches( - lambda x: pl.Series(stats.chi2.sf(x.to_numpy(), df=1)), - return_dtype=pl.Float64 - ).alias('P') - ) - - def qc_sumstat(self, file_path:str): - directory = self.uri + "_logs" - filename = os.path.basename(file_path) # "test.csv.gz" - # Remove all extensions - file_name = filename.split('.')[0] # "test" - - if not os.path.isdir(directory): - os.mkdir(directory) - sumstat_preqc = self.chunk_pl.to_pandas() - if self.type_sumstat == "gwas": - if self.type_trait== "quant": - sumstat_gl =gl.Sumstats(sumstat_preqc, - snpid="SNPID", - chrom="CHR", - pos="POS", - eaf="EAF", - beta="BETA", - se="SE", - p="P", - n="N", - ea = "EA", - nea = "NEA", - other = ["TRAIT","RSID"]) - else: - sumstat_gl =gl.Sumstats(sumstat_preqc, - snpid="SNPID", - chrom="CHR", - pos="POS", - eaf="EAF", - beta="BETA", - se="SE", - p="P", - n="N", - ncase = "N_CASES", - ncontrol = "N_CONTROLS", - ea = "EA", - nea = "NEA", - other = ["TRAIT","RSID"]) - - else: - sumstat_gl =gl.Sumstats(sumstat_preqc, - snpid="SNPID", - chrom="CHR", - pos="POS", - eaf="EAF", - beta="BETA", - se="SE", - p="P", - n="N", - other = ["CELL","GENE","RSID","DIST","PHENO_VAR"]) - #sumstat_gl.fix_id() - sumstat_gl.fix_chr(remove=True) - sumstat_gl.fix_pos(remove=True) - sumstat_gl.fix_allele(remove=True) - sumstat_gl.check_sanity() - sumstat_gl.check_data_consistency() - #sumstat_gl.remove_dup(mode="m") - #sumstat_gl.basic_check(n_cores = 4, remove=True, remove_dup=True) - - sumstat_gl.log.save(directory + "/" + file_name) - self.chunk_pl = pl.from_pandas(sumstat_gl.data) - - def ingest_data(self, file_path): - """Append harmonized data to TileDB.""" - pl.Config.set_tbl_cols(-1) - if self.type_sumstat == "gwas": - dedup_keys = ["CHR", "POS", "TRAIT"] - else: - dedup_keys = ["CHR", "POS", "CELL", "GENE"] - - # Option A (recommended): window count -> keep only rows whose group count == 1 - self.chunk_pl = ( - self.chunk_pl - .with_columns(pl.count().over(dedup_keys).alias("_grp_count")) - .filter(pl.col("_grp_count") == 1) - .drop("_grp_count") - ) - self.chunk_pl = self.chunk_pl.with_columns([ - pl.col("CHR").cast(pl.UInt16), - pl.col("POS").cast(pl.UInt32) - ]) - chunk_pl_ingest = self.chunk_pl.select(self.tiledb_types.keys()) - chunk_pl_ingest = chunk_pl_ingest.drop_nulls() - try: - tiledb.from_pandas( - uri=self.uri, - dataframe=chunk_pl_ingest.to_pandas(), - index_dims=self.dimension_tiledb, - column_types=self.tiledb_types, - allows_duplicates = False, - mode="append" - ) - logger.info(f"Successfully appended chunk to TileDB for file {file_path}") - except Exception as e: - logger.error(f"Failed to append chunk to TileDB for file {file_path}: {e}") - raise - - - def create_metadata(self, file_path: str): - """Create and store metadata as individual JSON files.""" - metadata = { - "traits": [], - "CELL": [] - } - self.chunk_pl = self.chunk_pl.select( - [col for col in self.chunk_pl.columns if self.chunk_pl[col].null_count() < self.chunk_pl.height] - ) - self.chunk_pl = self.chunk_pl.drop_nulls() - if self.type_sumstat == "qtl": - # Get unique cell types - celltypes = self.chunk_pl["CELL"].unique().to_list() - if not celltypes: - raise HarmonizationError("No cell types found in the data") - - # Update CELL list - metadata["CELL"] = celltypes - - # Process each cell type - for cell in celltypes: - # Filter by this cell type and compute ACAT per gene - df_cell = self.chunk_pl.filter(pl.col("CELL") == cell) - - # Group by CHR and GENE, compute ACAT for each group - chr_gene_agg = df_cell.group_by(["CHR", "GENE"]).agg([ - pl.col("P").map_batches( - lambda s: pl.Series([acat_optimized(s)]), - return_dtype=pl.Float64 - ).alias("ACAT_LIST"), - pl.col("P").min().alias("min_P"), - pl.col("N").first().alias("N"), - pl.col("PHENO_VAR").first().alias("PHENO_VAR") - ]) - chr_gene_agg = chr_gene_agg.with_columns( - pl.col("ACAT_LIST").list.first().alias("ACAT") - ) - - # Initialize cell structure if not exists - if cell not in metadata: - metadata[cell] = {} - - # Populate metadata with chromosome -> gene structure - for row in chr_gene_agg.iter_rows(named=True): - chrom = str(row["CHR"]) # Convert to string for consistency - gene = row["GENE"] - acat_val = row["ACAT"] - n_val = float(row["N"]) - min_p = float(row["min_P"]) - pheno_val = float(row["PHENO_VAR"]) - - if chrom not in metadata[cell]: - metadata[cell][chrom] = {} - - gene_metadata = { - "ACAT": float(acat_val), - "N": n_val, - "PHENO_VAR": pheno_val, - "MIN_P":min_p - } - - metadata[cell][chrom][gene] = gene_metadata - - else: # GWAS case - if "TRAIT" not in self.chunk_pl.columns: - raise HarmonizationError("TRAIT column is missing in the data") - - traits = self.chunk_pl["TRAIT"].unique().to_list() - metadata["traits"] = traits - - for trait in traits: - df_trait = self.chunk_pl.filter(pl.col("TRAIT") == trait) - pheno_var = compute_pheno_variance(df_trait, self.type_trait) - n_val = df_trait["N"].unique().to_list()[0] - min_p = df_trait["P"].min().to_list()[0] - - trait_metadata = { - "N": float(n_val), - "PHENO_VAR": float(pheno_var), - "MIN_P": float(min_p) - } - - if self.type_trait == "binary": - n_cases = df_trait["N_CASES"].unique().to_list()[0] - n_controls = df_trait["N_CONTROLS"].unique().to_list()[0] - trait_metadata.update({ - "N_CASES": float(n_cases), - "N_CONTROLS": float(n_controls) - }) - - metadata[trait] = trait_metadata - - # Write individual metadata file instead of updating TileDB - self._write_individual_metadata(metadata, file_path) - - logger.info(f"Individual metadata file created for {file_path}") - - def _write_individual_metadata(self, metadata, file_path): - """Write metadata to individual JSON file.""" - metadata_dir = f"{self.uri}_metadata_parts" - os.makedirs(metadata_dir, exist_ok=True) - - # Create a safe filename from the original file path - file_stem = Path(file_path).stem - metadata_file = os.path.join(metadata_dir, f"{file_stem}.json") - - with open(metadata_file, 'w') as f: - json.dump(metadata, f, indent=2) - - logger.info(f"Metadata written to {metadata_file}") - - def export_metadata_to_csv(self, output_path: str = None): - """Export metadata to CSV format.""" - if output_path is None: - output_path = f"{self.uri}_metadata.csv" - - # Get the merged metadata from TileDB - with tiledb.open(self.uri, "r") as array: - merged_metadata_json = array.meta.get("merged_metadata", "{}") - - if not merged_metadata_json: - logger.warning("No merged metadata found in TileDB array") - return - - merged_metadata = json.loads(merged_metadata_json) - - # Create rows for CSV - rows = [] - - # Process QTL data (cell types) - if "CELL" in merged_metadata and merged_metadata["CELL"]: - for cell_type in merged_metadata["CELL"]: - if cell_type in merged_metadata: - cell_data = merged_metadata[cell_type] - for chrom, genes in cell_data.items(): - for gene_id, gene_metadata in genes.items(): - row = { - "CHR": chrom, - "CELL": cell_type, - "GENE": gene_id, - "ACAT": gene_metadata.get("ACAT", ""), - "N": gene_metadata.get("N", ""), - "PHENO_VAR": gene_metadata.get("PHENO_VAR", ""), - "MIN_P": gene_metadata.get("MIN_P", "") - } - rows.append(row) - - # Process GWAS data (traits) - if "traits" in merged_metadata and merged_metadata["traits"]: - for trait in merged_metadata["traits"]: - if trait in merged_metadata: - trait_data = merged_metadata[trait] - row = { - "TRAIT": trait, - "N": trait_data.get("N", ""), - "PHENO_VAR": trait_data.get("PHENO_VAR", ""), - "MIN_P": trait_data.get("MIN_P", ""), - "N_CASES": trait_data.get("N_CASES", ""), - "N_CONTROLS": trait_data.get("N_CONTROLS", "") - } - rows.append(row) - - # Create DataFrame and save to CSV - if rows: - df = pd.DataFrame(rows) - - # Reorder columns for better readability - if "CELL" in df.columns: - # QTL format - column_order = ["CHR", "CELL", "GENE", "ACAT", "MIN_P", "N", "PHENO_VAR"] - # Only include columns that exist in the DataFrame - column_order = [col for col in column_order if col in df.columns] - df = df[column_order] - else: - # GWAS format - column_order = ["TRAIT", "N", "PHENO_VAR", "MIN_P", "N_CASES", "N_CONTROLS"] - column_order = [col for col in column_order if col in df.columns] - df = df[column_order] - - df.to_csv(output_path, index=False) - logger.info(f"Metadata exported to {output_path}") - - # Print summary - if "CELL" in df.columns: - logger.info(f"Exported {len(df)} gene-cell type combinations") - logger.info(f"Cell types: {df['CELL'].nunique()}") - logger.info(f"Genes: {df['GENE'].nunique()}") - logger.info(f"Chromosomes: {df['CHR'].nunique()}") - else: - logger.info(f"Exported {len(df)} traits") - - return df - else: - logger.warning("No metadata found to export") - return pd.DataFrame() - - def merge_metadata_files(self): - """Merge all individual metadata files into final TileDB metadata.""" - metadata_dir = f"{self.uri}_metadata_parts" - - if not os.path.exists(metadata_dir): - logger.warning(f"No metadata directory found at {metadata_dir}") - return - - merged_metadata = { - "traits": [], - "CELL": [] - } - - # Process all individual metadata files - metadata_files = list(Path(metadata_dir).glob("*.json")) - logger.info(f"Found {len(metadata_files)} metadata files to merge") - - for metadata_file in metadata_files: - try: - with open(metadata_file, 'r') as f: - file_metadata = json.load(f) - - # Merge traits (for GWAS) - if "traits" in file_metadata and file_metadata["traits"]: - current_traits = set(merged_metadata.get("traits", [])) - new_traits = set(file_metadata["traits"]) - merged_metadata["traits"] = list(current_traits.union(new_traits)) - - for trait in file_metadata["traits"]: - if trait in file_metadata: - if trait not in merged_metadata: - merged_metadata[trait] = file_metadata[trait] - else: - logger.warning(f"Trait {trait} already exists in metadata, overwriting") - merged_metadata[trait] = file_metadata[trait] - - # Merge CELL and cell metadata (for QTL) - if "CELL" in file_metadata and file_metadata["CELL"]: - current_cells = set(merged_metadata.get("CELL", [])) - new_cells = set(file_metadata["CELL"]) - merged_metadata["CELL"] = list(current_cells.union(new_cells)) - - for cell in file_metadata["CELL"]: - if cell in file_metadata: - if cell not in merged_metadata: - merged_metadata[cell] = {} - - for chrom, genes in file_metadata[cell].items(): - if chrom not in merged_metadata[cell]: - merged_metadata[cell][chrom] = {} - - for gene, gene_data in genes.items(): - if gene in merged_metadata[cell][chrom]: - logger.warning(f"Gene {gene} already exists in cell {cell} chromosome {chrom}, overwriting") - merged_metadata[cell][chrom][gene] = gene_data - - logger.info(f"Processed {metadata_file.name}") - - except Exception as e: - logger.error(f"Error processing metadata file {metadata_file}: {e}") - continue - - # Store the final merged metadata in TileDB - with tiledb.open(self.uri, "w") as array: - array.meta["merged_metadata"] = json.dumps(merged_metadata) - - # Export to CSV - self.export_metadata_to_csv() - - # Log summary - cell_count = len(merged_metadata.get("CELL", [])) - trait_count = len(merged_metadata.get("traits", [])) - - total_genes = 0 - for cell_type in merged_metadata.get("CELL", []): - if cell_type in merged_metadata: - for chrom_data in merged_metadata[cell_type].values(): - total_genes += len(chrom_data) - - logger.info(f"Final merged metadata: {cell_count} cell types, {trait_count} traits, {total_genes} total genes") - +"""Backward-compatibility shim. + +All ingestion logic has been moved to the +:mod:`tdbsumstat.utils.ingest` package, which is composed of focused +modules: + +* :mod:`tdbsumstat.utils.ingest.schema` – TileDB schema creation +* :mod:`tdbsumstat.utils.ingest.mapping` – column-mapping CSV parsing +* :mod:`tdbsumstat.utils.ingest.harmonize` – data normalisation +* :mod:`tdbsumstat.utils.ingest.qc` – optional gwaslab QC +* :mod:`tdbsumstat.utils.ingest.writer` – TileDB data writer +* :mod:`tdbsumstat.utils.ingest.metadata` – metadata management + +This file re-exports ``Harmonize`` and ``HarmonizationError`` so that +existing code that imports from ``tdbsumstat.utils.harmonize_ingest`` +continues to work without modification. +""" +from tdbsumstat.utils.ingest import Harmonize, HarmonizationError # noqa: F401 + +__all__ = ["Harmonize", "HarmonizationError"] diff --git a/tdbsumstat/utils/ingest/__init__.py b/tdbsumstat/utils/ingest/__init__.py new file mode 100644 index 0000000..4f7cd29 --- /dev/null +++ b/tdbsumstat/utils/ingest/__init__.py @@ -0,0 +1,92 @@ +"""Ingestion package for TileDB-sumstat. + +The ``Harmonize`` class is composed from focused mixin classes: + +* :class:`~tdbsumstat.utils.ingest.schema.SchemaMixin` – TileDB array creation +* :class:`~tdbsumstat.utils.ingest.mapping.MappingMixin` – column-mapping CSV parsing +* :class:`~tdbsumstat.utils.ingest.harmonize.HarmonizeMixin` – data normalisation +* :class:`~tdbsumstat.utils.ingest.qc.QCMixin` – optional gwaslab QC +* :class:`~tdbsumstat.utils.ingest.writer.WriterMixin` – TileDB data writer +* :class:`~tdbsumstat.utils.ingest.metadata.MetadataMixin` – metadata management +""" +import logging + +from tdbsumstat.utils.ingest.errors import HarmonizationError +from tdbsumstat.utils.ingest.harmonize import HarmonizeMixin +from tdbsumstat.utils.ingest.mapping import MappingMixin +from tdbsumstat.utils.ingest.metadata import MetadataMixin +from tdbsumstat.utils.ingest.qc import QCMixin +from tdbsumstat.utils.ingest.schema import SchemaMixin +from tdbsumstat.utils.ingest.writer import WriterMixin + +logger = logging.getLogger(__name__) + +__all__ = ["Harmonize", "HarmonizationError"] + + +class Harmonize(SchemaMixin, MappingMixin, HarmonizeMixin, QCMixin, WriterMixin, MetadataMixin): + """Pipeline class for harmonising and ingesting summary statistics into TileDB. + + Typical usage:: + + h = Harmonize(mapping_file="mapping.csv", uri="my_tiledb", + type_sumstat="qtl", pvar_file=None, + type_trait="quant", mac=None, maf=None, permuted=False) + + # One-time setup + h.create_tiledb() + h.create_mapping() + + # Per-file loop + for filepath, cell, gene in file_list: + sumstat = pl.read_csv(filepath, separator="\\t", null_values="NA") + h.harmonize(sumstat=sumstat, cell=cell, gene=gene, n=n, pheno_var=pheno_var) + h.ingest_data(file_path=filepath) + h.create_metadata(file_path=filepath) + + # Finalise + h.merge_metadata_files() + + Parameters + ---------- + mapping_file: + Path to a header-less CSV mapping source column names to TileDB names. + uri: + Path where the TileDB array is (or will be) stored. + type_sumstat: + ``"gwas"`` or ``"qtl"``. + pvar_file: + Optional path to a pvar file used to align alleles. + type_trait: + ``"quant"`` or ``"binary"`` (used for GWAS only). + mac: + Minimum minor allele count filter applied during harmonisation. + maf: + Minimum minor allele frequency filter applied during harmonisation. + permuted: + If ``True``, SE is derived from the permuted p-value rather than + being recalculated from BETA/SE. + """ + + def __init__( + self, + mapping_file: str, + uri: str, + type_sumstat: str, + pvar_file: str, + type_trait: str, + mac: int, + maf: float, + permuted: bool, + ) -> None: + self.mapping_file = mapping_file + self.uri = uri + self.pvar_file = pvar_file + self.type_sumstat = type_sumstat + self.type_trait = type_trait + self.mapping_types: dict = {} + self.tiledb_types: dict = {} + self.dimension_tiledb: list = [] + self.mac = mac + self.maf = maf + self.permuted = permuted diff --git a/tdbsumstat/utils/ingest/errors.py b/tdbsumstat/utils/ingest/errors.py new file mode 100644 index 0000000..bffac51 --- /dev/null +++ b/tdbsumstat/utils/ingest/errors.py @@ -0,0 +1,6 @@ +"""Custom exception types for the ingestion pipeline.""" + + +class HarmonizationError(Exception): + """Raised when data harmonization fails due to missing or invalid input.""" + pass diff --git a/tdbsumstat/utils/ingest/harmonize.py b/tdbsumstat/utils/ingest/harmonize.py new file mode 100644 index 0000000..90e6280 --- /dev/null +++ b/tdbsumstat/utils/ingest/harmonize.py @@ -0,0 +1,272 @@ +"""Data harmonization – column renaming, allele standardisation, p-value computation.""" +import logging +from pathlib import Path + +import numpy as np +import polars as pl +from scipy import stats + +logger = logging.getLogger(__name__) + + +class HarmonizeMixin: + """Mixin that normalises a summary statistics Polars DataFrame. + + After calling :meth:`harmonize`, the processed data is stored in + ``self.chunk_pl`` and column-type metadata is stored in + ``self.tiledb_types``. + """ + + # ------------------------------------------------------------------ + # Public API + # ------------------------------------------------------------------ + + def harmonize( + self, + sumstat: pl.DataFrame, + trait: str = None, + cell: str = None, + gene: str = None, + pheno_var: float = None, + n: int = None, + n_controls: int = None, + n_cases: int = None, + ) -> None: + """Rename columns, compute missing fields, and set TileDB types. + + Parameters + ---------- + sumstat: + Raw Polars DataFrame to harmonise. + trait: + Trait identifier (GWAS only). + cell: + Cell-type identifier (QTL only). + gene: + Gene identifier (QTL only). + pheno_var: + Phenotypic variance (quant trait). + n: + Sample size. + n_controls: + Number of controls (binary GWAS). + n_cases: + Number of cases (binary GWAS). + + Raises + ------ + HarmonizationError + When required columns are missing and cannot be inferred. + """ + from tdbsumstat.utils.ingest.errors import HarmonizationError + + self.chunk_pl = sumstat.rename(self.mapping_types) + + # Extract CHR/POS from SNPID if not present + if "CHR" not in self.chunk_pl.columns or "POS" not in self.chunk_pl.columns: + self.chunk_pl = self.chunk_pl.with_columns( + pl.col("SNPID") + .str.split_exact(":", 4) + .struct.rename_fields(["CHR", "POS", "A1", "A2"]) + .alias("fields") + ).unnest("fields") + + if "SNPID" in self.chunk_pl.columns: + self.chunk_pl = self.chunk_pl.drop("SNPID") + if "EAF" not in self.chunk_pl.columns: + self.chunk_pl = self.chunk_pl.with_columns(pl.lit(0).alias("EAF")) + if "DIST" not in self.chunk_pl.columns: + self.chunk_pl = self.chunk_pl.with_columns(pl.lit(1).alias("DIST")) + + # Optional MAF filter + if self.maf is not None: + self.chunk_pl = self.chunk_pl.with_columns( + pl.min_horizontal(pl.col("EAF"), 1 - pl.col("EAF")).alias("MAF") + ).filter(pl.col("MAF") >= self.maf) + + # Sample-size and phenotypic-variance handling + self._apply_sample_size(n, n_cases, n_controls, pheno_var) + + # Second SNPID-drop guard (in case rename created one) + if "SNPID" in self.chunk_pl.columns: + self.chunk_pl = self.chunk_pl.drop("SNPID") + + # Allele standardisation: A1 < A2 order + self._standardise_alleles() + + # Assign RSID placeholder if absent + if "RSID" not in self.chunk_pl.columns: + self.chunk_pl = self.chunk_pl.with_columns(pl.lit("None").alias("RSID")) + + # Convert LOG10P → P + if "LOG10P" in self.chunk_pl.columns: + self.chunk_pl = self.chunk_pl.with_columns( + (10 ** (-pl.col("LOG10P"))).alias("P") + ) + + # Compute / recalculate p-value + if self.permuted: + self.chunk_pl = self.chunk_pl.with_columns( + ( + pl.col("BETA").pow(2) + / pl.col("P").map_batches( + lambda x: pl.Series(stats.chi2.isf(x.to_numpy(), df=1)) + ) + ).sqrt().alias("SE") + ) + else: + self.chunk_pl = self.chunk_pl.drop("P") + self.chunk_pl = self.chunk_pl.with_columns( + (pl.col("BETA") / pl.col("SE")) + .pow(2) + .map_batches( + lambda x: pl.Series(stats.chi2.sf(x.to_numpy(), df=1)), + return_dtype=pl.Float64, + ) + .alias("P") + ) + + # Set dimension labels and TileDB type maps + self._set_tiledb_types(trait=trait, cell=cell, gene=gene) + + def align_alleles(self) -> None: + """Align effect/other alleles using an external pvar reference file. + + Requires ``self.pvar_file`` to be set and point to a valid file. + + Raises + ------ + HarmonizationError + If ``pvar_file`` is not provided. + FileNotFoundError + If the pvar file path does not exist. + """ + from tdbsumstat.utils.ingest.errors import HarmonizationError + + if not self.pvar_file: + raise HarmonizationError("pvar_file must be provided to verify alleles order") + if not Path(self.pvar_file).is_file(): + raise FileNotFoundError(f"pvar_file {self.pvar_file} does not exist") + + pvar_df = pl.read_csv( + self.pvar_file, + separator="\t", + has_header=True, + schema_overrides={"CHROM": pl.Utf8, "POS": pl.Utf8, "SNPID": pl.Utf8, + "REF": pl.Utf8, "ALT": pl.Utf8}, + ) + self.chunk_pl = self.chunk_pl.join(pvar_df, on="SNPID", how="inner", suffix="_pvar") + + swap = pl.col("ALT") < pl.col("REF") + self.chunk_pl = self.chunk_pl.with_columns([ + pl.when(swap).then(pl.col("BETA")).otherwise(-pl.col("BETA")), + pl.when(swap).then(pl.col("EAF")).otherwise(1.0 - pl.col("EAF")), + ]) + + # ------------------------------------------------------------------ + # Private helpers + # ------------------------------------------------------------------ + + def _apply_sample_size(self, n, n_cases, n_controls, pheno_var): + """Add N, N_CASES, N_CONTROLS columns and apply MAC filter.""" + from tdbsumstat.utils.ingest.errors import HarmonizationError + + if self.type_trait == "quant": + if "N" not in self.chunk_pl.columns: + if n is not None: + self.chunk_pl = self.chunk_pl.with_columns(pl.lit(int(n)).alias("N")) + else: + raise HarmonizationError("N column is missing and N parameter is not provided") + if self.mac is not None: + self.chunk_pl = self.chunk_pl.with_columns( + (2 * pl.col("N") * pl.min_horizontal(pl.col("EAF"), 1 - pl.col("EAF"))).alias("MAC") + ).filter(pl.col("MAC") >= self.mac) + self.chunk_pl = self.chunk_pl.with_columns(pl.lit(pheno_var).alias("PHENO_VAR")) + + elif self.type_trait == "binary": + if not all(col in self.chunk_pl.columns for col in ["N_CASES", "N_CONTROLS"]): + if n_cases is not None and n_controls is not None: + self.chunk_pl = self.chunk_pl.with_columns( + pl.lit(float(n_cases)).alias("N_CASES"), + pl.lit(float(n_controls)).alias("N_CONTROLS"), + pl.lit(float(n_cases) + float(n_controls)).alias("N"), + ) + else: + raise HarmonizationError("n_cases and n_controls columns are missing and were not provided") + if self.mac is not None: + self.chunk_pl = self.chunk_pl.with_columns( + (2 * pl.col("N") * pl.min_horizontal(pl.col("EAF"), 1 - pl.col("EAF"))).alias("MAC") + ).filter(pl.col("MAC") >= self.mac) + else: + from tdbsumstat.utils.ingest.errors import HarmonizationError + raise HarmonizationError("Type of trait must be either binary or quant") + + def _standardise_alleles(self): + """Sort A1/A2 lexicographically; flip BETA and EAF when swapped.""" + swap = pl.col("A1") < pl.col("A2") + + self.chunk_pl = self.chunk_pl.with_columns([ + pl.when(swap).then(pl.col("A1")).otherwise(pl.col("A2")).alias("EA"), + pl.when(swap).then(pl.col("A2")).otherwise(pl.col("A1")).alias("NEA"), + ]) + + self.chunk_pl = self.chunk_pl.with_columns( + pl.concat_str( + [ + pl.col("CHR"), + pl.col("POS").cast(pl.Utf8), + pl.col("EA"), + pl.col("NEA"), + ], + separator=":", + ).alias("SNPID") + ) + + if self.pvar_file: + self.align_alleles() + self.chunk_pl = self.chunk_pl.drop(["REF", "ALT"]) + else: + self.chunk_pl = self.chunk_pl.with_columns([ + pl.when(swap).then(-pl.col("BETA")).otherwise(pl.col("BETA")).alias("BETA"), + pl.when(swap).then(1.0 - pl.col("EAF")).otherwise(pl.col("EAF")).alias("EAF"), + ]) + + self.chunk_pl = self.chunk_pl.drop(["A1", "A2"]) + self.chunk_pl = self.chunk_pl.with_columns( + pl.concat_str(pl.lit("chr"), pl.col("SNPID")).alias("SNPID") + ) + + def _set_tiledb_types(self, trait, cell, gene): + """Populate ``self.tiledb_types`` and add dimension columns.""" + if self.type_sumstat == "gwas": + self.tiledb_types = { + "CHR": np.uint16, + "TRAIT": str, + "POS": np.uint32, + "SNPID": str, + "RSID": str, + "EAF": np.float32, + "BETA": np.float32, + "SE": np.float32, + "P": np.float64, + } + if "TRAIT" not in self.chunk_pl.columns: + self.chunk_pl = self.chunk_pl.with_columns(pl.lit(trait).alias("TRAIT")) + else: + self.tiledb_types = { + "CHR": np.uint16, + "CELL": str, + "GENE": str, + "POS": np.uint32, + "SNPID": str, + "RSID": str, + "DIST": np.int64, + "EAF": np.float32, + "BETA": np.float32, + "SE": np.float32, + "P": np.float64, + } + if "CELL" not in self.chunk_pl.columns: + self.chunk_pl = self.chunk_pl.with_columns(pl.lit(cell).alias("CELL")) + if "GENE" not in self.chunk_pl.columns: + self.chunk_pl = self.chunk_pl.with_columns(pl.lit(gene).alias("GENE")) diff --git a/tdbsumstat/utils/ingest/mapping.py b/tdbsumstat/utils/ingest/mapping.py new file mode 100644 index 0000000..7b6df0d --- /dev/null +++ b/tdbsumstat/utils/ingest/mapping.py @@ -0,0 +1,41 @@ +"""Column-mapping utilities for summary statistics ingestion.""" +import logging + +import pandas as pd + +logger = logging.getLogger(__name__) + + +class MappingMixin: + """Mixin that parses and validates the column-mapping CSV file.""" + + def create_mapping(self) -> None: + """Load the mapping file and populate ``self.mapping_types``. + + The mapping file is a header-less CSV with two columns: + ``source_column_name, target_column_name``. + + Required target columns: ``BETA``, ``SE``. + Required targets for position: either ``CHR`` + ``POS``, or ``SNPID``. + + Raises + ------ + HarmonizationError + If the file is empty, or required columns are missing. + """ + from tdbsumstat.utils.ingest.errors import HarmonizationError + + df = pd.read_csv(self.mapping_file, header=None, names=["key", "value"]) + if df.empty: + raise HarmonizationError("Mapping file is empty or not formatted correctly.") + + self.mapping_types = dict(zip(df["key"], df["value"])) + + if not all(col in self.mapping_types.values() for col in ["BETA", "SE"]): + raise HarmonizationError("Mapping file must contain BETA and SE columns.") + + if not all(col in self.mapping_types.values() for col in ["CHR", "POS"]): + if "SNPID" not in self.mapping_types.values(): + raise HarmonizationError("Mapping file must contain either CHR, and POS or SNPID columns.") + + logger.info("Column mapping loaded: %d column(s) mapped", len(self.mapping_types)) diff --git a/tdbsumstat/utils/ingest/metadata.py b/tdbsumstat/utils/ingest/metadata.py new file mode 100644 index 0000000..a90086b --- /dev/null +++ b/tdbsumstat/utils/ingest/metadata.py @@ -0,0 +1,284 @@ +"""Metadata creation, export and merging for the ingestion pipeline.""" +import json +import logging +import os +from pathlib import Path + +import pandas as pd +import polars as pl +import tiledb + +from tdbsumstat.utils import acat_optimized, compute_pheno_variance + +logger = logging.getLogger(__name__) + + +class MetadataMixin: + """Mixin that handles per-file and merged TileDB metadata.""" + + # ------------------------------------------------------------------ + # Per-file metadata + # ------------------------------------------------------------------ + + def create_metadata(self, file_path: str) -> None: + """Compute summary statistics and write a per-file JSON metadata fragment. + + The fragment is saved under ``{self.uri}_metadata_parts/``. + + Parameters + ---------- + file_path: + Original file path; used to derive the metadata filename. + """ + from tdbsumstat.utils.ingest.errors import HarmonizationError + + metadata: dict = {"traits": [], "CELL": []} + + # Drop fully-null columns, then drop null rows + self.chunk_pl = self.chunk_pl.select( + [col for col in self.chunk_pl.columns if self.chunk_pl[col].null_count() < self.chunk_pl.height] + ) + self.chunk_pl = self.chunk_pl.drop_nulls() + + if self.type_sumstat == "qtl": + metadata = self._build_qtl_metadata(metadata) + else: + metadata = self._build_gwas_metadata(metadata) + + self._write_individual_metadata(metadata, file_path) + logger.info("Individual metadata file created for %s", file_path) + + def _build_qtl_metadata(self, metadata: dict) -> dict: + """Populate *metadata* with per-cell-type, per-gene statistics.""" + from tdbsumstat.utils.ingest.errors import HarmonizationError + + celltypes = self.chunk_pl["CELL"].unique().to_list() + if not celltypes: + raise HarmonizationError("No cell types found in the data") + + metadata["CELL"] = celltypes + + for cell in celltypes: + df_cell = self.chunk_pl.filter(pl.col("CELL") == cell) + + chr_gene_agg = df_cell.group_by(["CHR", "GENE"]).agg([ + pl.col("P").map_batches( + lambda s: pl.Series([acat_optimized(s)]), + return_dtype=pl.Float64, + ).alias("ACAT_LIST"), + pl.col("P").min().alias("min_P"), + pl.col("N").first().alias("N"), + pl.col("PHENO_VAR").first().alias("PHENO_VAR"), + ]) + chr_gene_agg = chr_gene_agg.with_columns( + pl.col("ACAT_LIST").list.first().alias("ACAT") + ) + + if cell not in metadata: + metadata[cell] = {} + + for row in chr_gene_agg.iter_rows(named=True): + chrom = str(row["CHR"]) + gene = row["GENE"] + metadata[cell].setdefault(chrom, {}) + metadata[cell][chrom][gene] = { + "ACAT": float(row["ACAT"]), + "N": float(row["N"]), + "PHENO_VAR": float(row["PHENO_VAR"]), + "MIN_P": float(row["min_P"]), + } + + return metadata + + def _build_gwas_metadata(self, metadata: dict) -> dict: + """Populate *metadata* with per-trait statistics.""" + from tdbsumstat.utils.ingest.errors import HarmonizationError + + if "TRAIT" not in self.chunk_pl.columns: + raise HarmonizationError("TRAIT column is missing in the data") + + traits = self.chunk_pl["TRAIT"].unique().to_list() + metadata["traits"] = traits + + for trait in traits: + df_trait = self.chunk_pl.filter(pl.col("TRAIT") == trait) + pheno_var = compute_pheno_variance(df_trait, self.type_trait) + n_val = df_trait["N"].unique().to_list()[0] + min_p = df_trait["P"].min().to_list()[0] + + trait_metadata: dict = { + "N": float(n_val), + "PHENO_VAR": float(pheno_var), + "MIN_P": float(min_p), + } + + if self.type_trait == "binary": + trait_metadata["N_CASES"] = float(df_trait["N_CASES"].unique().to_list()[0]) + trait_metadata["N_CONTROLS"] = float(df_trait["N_CONTROLS"].unique().to_list()[0]) + + metadata[trait] = trait_metadata + + return metadata + + def _write_individual_metadata(self, metadata: dict, file_path: str) -> None: + """Persist a metadata dict as a JSON fragment under the metadata_parts dir.""" + metadata_dir = f"{self.uri}_metadata_parts" + os.makedirs(metadata_dir, exist_ok=True) + + file_stem = Path(file_path).stem + metadata_file = os.path.join(metadata_dir, f"{file_stem}.json") + + with open(metadata_file, "w") as fh: + json.dump(metadata, fh, indent=2) + + logger.info("Metadata written to %s", metadata_file) + + # ------------------------------------------------------------------ + # Metadata export + # ------------------------------------------------------------------ + + def export_metadata_to_csv(self, output_path: str = None) -> pd.DataFrame: + """Export the merged TileDB metadata to a CSV file. + + Parameters + ---------- + output_path: + Destination path. Defaults to ``{self.uri}_metadata.csv``. + + Returns + ------- + pd.DataFrame + The exported metadata as a DataFrame (empty if nothing found). + """ + if output_path is None: + output_path = f"{self.uri}_metadata.csv" + + with tiledb.open(self.uri, "r") as array: + merged_metadata_json = array.meta.get("merged_metadata", "{}") + + if not merged_metadata_json: + logger.warning("No merged metadata found in TileDB array") + return pd.DataFrame() + + merged_metadata = json.loads(merged_metadata_json) + rows = [] + + # QTL data + for cell_type in merged_metadata.get("CELL", []): + cell_data = merged_metadata.get(cell_type, {}) + for chrom, genes in cell_data.items(): + for gene_id, gm in genes.items(): + rows.append({ + "CHR": chrom, "CELL": cell_type, "GENE": gene_id, + "ACAT": gm.get("ACAT", ""), + "N": gm.get("N", ""), + "PHENO_VAR": gm.get("PHENO_VAR", ""), + "MIN_P": gm.get("MIN_P", ""), + }) + + # GWAS data + for trait in merged_metadata.get("traits", []): + td = merged_metadata.get(trait, {}) + rows.append({ + "TRAIT": trait, + "N": td.get("N", ""), + "PHENO_VAR": td.get("PHENO_VAR", ""), + "MIN_P": td.get("MIN_P", ""), + "N_CASES": td.get("N_CASES", ""), + "N_CONTROLS": td.get("N_CONTROLS", ""), + }) + + if not rows: + logger.warning("No metadata found to export") + return pd.DataFrame() + + df = pd.DataFrame(rows) + + if "CELL" in df.columns: + col_order = [c for c in ["CHR", "CELL", "GENE", "ACAT", "MIN_P", "N", "PHENO_VAR"] if c in df.columns] + else: + col_order = [c for c in ["TRAIT", "N", "PHENO_VAR", "MIN_P", "N_CASES", "N_CONTROLS"] if c in df.columns] + + df = df[col_order] + df.to_csv(output_path, index=False) + logger.info("Metadata exported to %s", output_path) + return df + + # ------------------------------------------------------------------ + # Merge all per-file fragments into final TileDB metadata + # ------------------------------------------------------------------ + + def merge_metadata_files(self) -> None: + """Merge all per-file JSON fragments and store in TileDB. + + Reads every ``*.json`` from ``{self.uri}_metadata_parts/``, + merges them into a single dict, stores it in ``TileDB.meta`` + under the key ``merged_metadata``, and also exports a CSV. + """ + metadata_dir = f"{self.uri}_metadata_parts" + + if not os.path.exists(metadata_dir): + logger.warning("No metadata directory found at %s", metadata_dir) + return + + merged_metadata: dict = {"traits": [], "CELL": []} + + metadata_files = list(Path(metadata_dir).glob("*.json")) + logger.info("Found %d metadata file(s) to merge", len(metadata_files)) + + for metadata_file in metadata_files: + try: + with open(metadata_file, "r") as fh: + file_metadata = json.load(fh) + self._merge_single_metadata(merged_metadata, file_metadata) + logger.info("Processed %s", metadata_file.name) + except Exception as exc: + logger.error("Error processing metadata file %s: %s", metadata_file, exc) + continue + + with tiledb.open(self.uri, "w") as array: + array.meta["merged_metadata"] = json.dumps(merged_metadata) + + self.export_metadata_to_csv() + + cell_count = len(merged_metadata.get("CELL", [])) + trait_count = len(merged_metadata.get("traits", [])) + total_genes = sum( + len(chrom_data) + for cell_type in merged_metadata.get("CELL", []) + for chrom_data in merged_metadata.get(cell_type, {}).values() + ) + logger.info( + "Final merged metadata: %d cell type(s), %d trait(s), %d total gene(s)", + cell_count, trait_count, total_genes, + ) + + @staticmethod + def _merge_single_metadata(merged: dict, file_meta: dict) -> None: + """Merge one file_meta fragment into *merged* in-place.""" + # GWAS traits + if file_meta.get("traits"): + existing = set(merged.get("traits", [])) + merged["traits"] = list(existing.union(set(file_meta["traits"]))) + for trait in file_meta["traits"]: + if trait in file_meta: + if trait in merged: + logger.warning("Trait %s already exists – overwriting", trait) + merged[trait] = file_meta[trait] + + # QTL cell types + if file_meta.get("CELL"): + existing = set(merged.get("CELL", [])) + merged["CELL"] = list(existing.union(set(file_meta["CELL"]))) + for cell in file_meta["CELL"]: + if cell in file_meta: + merged.setdefault(cell, {}) + for chrom, genes in file_meta[cell].items(): + merged[cell].setdefault(chrom, {}) + for gene, gene_data in genes.items(): + if gene in merged[cell][chrom]: + logger.warning( + "Gene %s in cell %s chr %s already exists – overwriting", + gene, cell, chrom, + ) + merged[cell][chrom][gene] = gene_data diff --git a/tdbsumstat/utils/ingest/qc.py b/tdbsumstat/utils/ingest/qc.py new file mode 100644 index 0000000..ed08a9a --- /dev/null +++ b/tdbsumstat/utils/ingest/qc.py @@ -0,0 +1,69 @@ +"""GWAS-lab quality-control step (optional).""" +import logging +import os + +import polars as pl + +logger = logging.getLogger(__name__) + + +class QCMixin: + """Mixin that provides optional gwaslab-based QC on ``self.chunk_pl``.""" + + def qc_sumstat(self, file_path: str) -> None: + """Run gwaslab QC checks on the harmonised data in ``self.chunk_pl``. + + Logs are saved to ``{self.uri}_logs/{file_stem}.log``. + On success, ``self.chunk_pl`` is replaced with the QC-filtered + gwaslab DataFrame (converted back to Polars). + + Parameters + ---------- + file_path: + Original file path; used only to derive a log file name. + """ + import gwaslab as gl + + directory = self.uri + "_logs" + filename = os.path.basename(file_path) + file_name = filename.split(".")[0] + + os.makedirs(directory, exist_ok=True) + + sumstat_preqc = self.chunk_pl.to_pandas() + + if self.type_sumstat == "gwas": + if self.type_trait == "quant": + sumstat_gl = gl.Sumstats( + sumstat_preqc, + snpid="SNPID", chrom="CHR", pos="POS", + eaf="EAF", beta="BETA", se="SE", p="P", n="N", + ea="EA", nea="NEA", + other=["TRAIT", "RSID"], + ) + else: + sumstat_gl = gl.Sumstats( + sumstat_preqc, + snpid="SNPID", chrom="CHR", pos="POS", + eaf="EAF", beta="BETA", se="SE", p="P", n="N", + ncase="N_CASES", ncontrol="N_CONTROLS", + ea="EA", nea="NEA", + other=["TRAIT", "RSID"], + ) + else: + sumstat_gl = gl.Sumstats( + sumstat_preqc, + snpid="SNPID", chrom="CHR", pos="POS", + eaf="EAF", beta="BETA", se="SE", p="P", n="N", + other=["CELL", "GENE", "RSID", "DIST", "PHENO_VAR"], + ) + + sumstat_gl.fix_chr(remove=True) + sumstat_gl.fix_pos(remove=True) + sumstat_gl.fix_allele(remove=True) + sumstat_gl.check_sanity() + sumstat_gl.check_data_consistency() + + sumstat_gl.log.save(os.path.join(directory, file_name)) + self.chunk_pl = pl.from_pandas(sumstat_gl.data) + logger.info("QC completed for %s", file_path) diff --git a/tdbsumstat/utils/ingest/schema.py b/tdbsumstat/utils/ingest/schema.py new file mode 100644 index 0000000..cff4149 --- /dev/null +++ b/tdbsumstat/utils/ingest/schema.py @@ -0,0 +1,80 @@ +"""TileDB array schema creation for summary statistics ingestion.""" +import logging + +import numpy as np +import tiledb + +logger = logging.getLogger(__name__) + + +class SchemaMixin: + """Mixin that provides TileDB array schema creation.""" + + def create_tiledb(self) -> None: + """Create the TileDB array with the appropriate schema. + + The schema varies by ``self.type_sumstat``: + + * ``"gwas"`` – dimensions: CHR, TRAIT, POS + * ``"qtl"`` – dimensions: CHR, CELL, GENE, POS (+ DIST attribute) + """ + pos_domain = (1, 300_000_000) + chr_domain = (1, 24) + + attrs = [ + tiledb.Attr(name="SNPID", dtype="ascii", filters=tiledb.FilterList([tiledb.ZstdFilter(level=5)])), + tiledb.Attr(name="RSID", dtype="ascii", filters=tiledb.FilterList([tiledb.ZstdFilter(level=5)])), + tiledb.Attr(name="EAF", dtype=np.float32, filters=tiledb.FilterList([tiledb.ZstdFilter(level=5)])), + tiledb.Attr(name="BETA", dtype=np.float32, filters=tiledb.FilterList([tiledb.ZstdFilter(level=5)])), + tiledb.Attr(name="SE", dtype=np.float32, filters=tiledb.FilterList([tiledb.ZstdFilter(level=5)])), + tiledb.Attr(name="P", dtype=np.float64, filters=tiledb.FilterList([tiledb.ZstdFilter(level=5)])), + ] + + if self.type_sumstat == "gwas": + self.dimension_tiledb = ["CHR", "TRAIT", "POS"] + dom = tiledb.Domain( + tiledb.Dim( + name="CHR", domain=chr_domain, dtype=np.uint16, + filters=tiledb.FilterList([tiledb.ZstdFilter(level=5)]), + ), + tiledb.Dim( + name="TRAIT", dtype="ascii", var=True, + filters=tiledb.FilterList([tiledb.ZstdFilter(level=5)]), + ), + tiledb.Dim( + name="POS", domain=pos_domain, dtype=np.uint32, + filters=tiledb.FilterList([tiledb.ZstdFilter(level=5)]), + ), + ) + else: + self.dimension_tiledb = ["CHR", "CELL", "GENE", "POS"] + dom = tiledb.Domain( + tiledb.Dim( + name="CHR", domain=chr_domain, dtype=np.uint16, + filters=tiledb.FilterList([tiledb.ZstdFilter(level=5)]), + ), + tiledb.Dim( + name="CELL", dtype="ascii", var=True, + filters=tiledb.FilterList([tiledb.ZstdFilter(level=5)]), + ), + tiledb.Dim( + name="GENE", dtype="ascii", var=True, + filters=tiledb.FilterList([tiledb.ZstdFilter(level=5)]), + ), + tiledb.Dim( + name="POS", domain=pos_domain, dtype=np.uint32, + filters=tiledb.FilterList([tiledb.ZstdFilter(level=5)]), + ), + ) + attrs = attrs + [ + tiledb.Attr(name="DIST", dtype=np.float32, filters=tiledb.FilterList([tiledb.ZstdFilter(level=5)])), + ] + + schema = tiledb.ArraySchema( + domain=dom, + attrs=attrs, + sparse=True, + allows_duplicates=False, + ) + tiledb.Array.create(self.uri, schema) + logger.info("TileDB array created at %s (type_sumstat=%s)", self.uri, self.type_sumstat) diff --git a/tdbsumstat/utils/ingest/writer.py b/tdbsumstat/utils/ingest/writer.py new file mode 100644 index 0000000..04790da --- /dev/null +++ b/tdbsumstat/utils/ingest/writer.py @@ -0,0 +1,61 @@ +"""TileDB data writer – appends harmonised data to an existing TileDB array.""" +import logging + +import polars as pl +import tiledb + +logger = logging.getLogger(__name__) + + +class WriterMixin: + """Mixin that deduplicates and writes ``self.chunk_pl`` into TileDB.""" + + def ingest_data(self, file_path: str) -> None: + """Deduplicate and append ``self.chunk_pl`` to the TileDB array. + + Rows with duplicate index-dimension keys are removed before writing. + Only the columns declared in ``self.tiledb_types`` are written. + + Parameters + ---------- + file_path: + Original file path; used only for logging messages. + + Raises + ------ + Exception + Re-raises any exception from ``tiledb.from_pandas``. + """ + pl.Config.set_tbl_cols(-1) + + dedup_keys = ["CHR", "POS", "TRAIT"] if self.type_sumstat == "gwas" else ["CHR", "POS", "CELL", "GENE"] + + # Drop ALL rows that share an index key with at least one other row. + # Rows with ambiguous (duplicated) positions are treated as unreliable + # and excluded from the TileDB write entirely. + self.chunk_pl = ( + self.chunk_pl + .with_columns(pl.len().over(dedup_keys).alias("_grp_count")) + .filter(pl.col("_grp_count") == 1) + .drop("_grp_count") + ) + self.chunk_pl = self.chunk_pl.with_columns([ + pl.col("CHR").cast(pl.UInt16), + pl.col("POS").cast(pl.UInt32), + ]) + + chunk_pl_ingest = self.chunk_pl.select(self.tiledb_types.keys()).drop_nulls() + + try: + tiledb.from_pandas( + uri=self.uri, + dataframe=chunk_pl_ingest.to_pandas(), + index_dims=self.dimension_tiledb, + column_types=self.tiledb_types, + allows_duplicates=False, + mode="append", + ) + logger.info("Successfully appended chunk to TileDB for file %s", file_path) + except Exception as exc: + logger.error("Failed to append chunk to TileDB for file %s: %s", file_path, exc) + raise diff --git a/tests/test_ingest.py b/tests/test_ingest.py new file mode 100644 index 0000000..9f75c4b --- /dev/null +++ b/tests/test_ingest.py @@ -0,0 +1,391 @@ +"""Tests for the refactored ingestion modules under tdbsumstat/utils/ingest/.""" +import json +import os + +import polars as pl +import pytest +import tiledb + +from tdbsumstat.utils.ingest import Harmonize, HarmonizationError +from tdbsumstat.utils.ingest.errors import HarmonizationError as DirectError +from tdbsumstat.utils.ingest.schema import SchemaMixin +from tdbsumstat.utils.ingest.mapping import MappingMixin +from tdbsumstat.utils.ingest.harmonize import HarmonizeMixin +from tdbsumstat.utils.ingest.writer import WriterMixin +from tdbsumstat.utils.ingest.metadata import MetadataMixin + +EXAMPLE_DATA_DIR = os.path.join(os.path.dirname(__file__), "..", "example_data") + + +# --------------------------------------------------------------------------- +# Helpers / fixtures +# --------------------------------------------------------------------------- + +@pytest.fixture() +def mapping_file(): + return os.path.join(EXAMPLE_DATA_DIR, "mapping_file_test.csv") + + +@pytest.fixture() +def sample_qtl_df(): + """Minimal QTL Polars DataFrame matching the mapping_file_test column names.""" + return pl.DataFrame({ + "Chr": [20, 20, 20], + "Gene": ["ENSG0000010000"] * 3, + "cell.type": ["T_gd"] * 3, + "pos": [100, 200, 300], + "a0": ["A", "C", "G"], + "a1": ["T", "G", "A"], + "p": [0.01, 0.5, 0.001], + "N": [500, 500, 500], + "beta": [0.1, -0.2, 0.3], + "se": [0.05, 0.08, 0.04], + }) + + +@pytest.fixture() +def empty_qtl_df(): + """Empty QTL Polars DataFrame with correct schema.""" + return pl.DataFrame({ + "Chr": pl.Series([], dtype=pl.Int64), + "Gene": pl.Series([], dtype=pl.Utf8), + "cell.type": pl.Series([], dtype=pl.Utf8), + "pos": pl.Series([], dtype=pl.Int64), + "a0": pl.Series([], dtype=pl.Utf8), + "a1": pl.Series([], dtype=pl.Utf8), + "p": pl.Series([], dtype=pl.Float64), + "N": pl.Series([], dtype=pl.Int64), + "beta": pl.Series([], dtype=pl.Float64), + "se": pl.Series([], dtype=pl.Float64), + }) + + +# --------------------------------------------------------------------------- +# SchemaMixin +# --------------------------------------------------------------------------- + +class TestSchemaMixin: + def test_qtl_dimensions(self, mapping_file, tmp_path): + uri = str(tmp_path / "qtl") + h = Harmonize(mapping_file, uri, "qtl", None, "quant", None, None, False) + h.create_tiledb() + schema = tiledb.ArraySchema.load(uri) + dim_names = [schema.domain.dim(i).name for i in range(schema.domain.ndim)] + assert dim_names == ["CHR", "CELL", "GENE", "POS"] + attr_names = [schema.attr(i).name for i in range(schema.nattr)] + assert "DIST" in attr_names + + def test_gwas_dimensions(self, mapping_file, tmp_path): + uri = str(tmp_path / "gwas") + h = Harmonize(mapping_file, uri, "gwas", None, "quant", None, None, False) + h.create_tiledb() + schema = tiledb.ArraySchema.load(uri) + dim_names = [schema.domain.dim(i).name for i in range(schema.domain.ndim)] + assert dim_names == ["CHR", "TRAIT", "POS"] + attr_names = [schema.attr(i).name for i in range(schema.nattr)] + assert "DIST" not in attr_names + + def test_sparse_schema(self, mapping_file, tmp_path): + uri = str(tmp_path / "sparse_check") + h = Harmonize(mapping_file, uri, "qtl", None, "quant", None, None, False) + h.create_tiledb() + schema = tiledb.ArraySchema.load(uri) + assert schema.sparse is True + + +# --------------------------------------------------------------------------- +# MappingMixin +# --------------------------------------------------------------------------- + +class TestMappingMixin: + def test_creates_dict(self, mapping_file): + h = Harmonize(mapping_file, "dummy", "qtl", None, "quant", None, None, False) + h.create_mapping() + assert isinstance(h.mapping_types, dict) + assert len(h.mapping_types) > 0 + + def test_missing_beta_raises(self, tmp_path): + f = tmp_path / "m.csv" + f.write_text("Chr,CHR\npos,POS\n") + h = Harmonize(str(f), "dummy", "qtl", None, "quant", None, None, False) + with pytest.raises(HarmonizationError, match="BETA and SE"): + h.create_mapping() + + def test_empty_file_raises(self, tmp_path): + f = tmp_path / "empty.csv" + f.write_text("") + h = Harmonize(str(f), "dummy", "qtl", None, "quant", None, None, False) + with pytest.raises(HarmonizationError, match="empty or not formatted"): + h.create_mapping() + + def test_harmonization_error_importable_directly(self): + assert DirectError is HarmonizationError + + +# --------------------------------------------------------------------------- +# HarmonizeMixin +# --------------------------------------------------------------------------- + +class TestHarmonizeMixin: + def test_snpid_prefixed_with_chr(self, mapping_file, sample_qtl_df): + h = Harmonize(mapping_file, "dummy", "qtl", None, "quant", None, None, False) + h.create_mapping() + h.harmonize(sumstat=sample_qtl_df, cell="T_gd", gene="ENSG0000010000", n=500, pheno_var=1.5) + for snpid in h.chunk_pl["SNPID"].to_list(): + assert snpid.startswith("chr"), f"SNPID {snpid!r} does not start with 'chr'" + + def test_cell_and_gene_columns_added(self, mapping_file, sample_qtl_df): + h = Harmonize(mapping_file, "dummy", "qtl", None, "quant", None, None, False) + h.create_mapping() + h.harmonize(sumstat=sample_qtl_df, cell="T_gd", gene="ENSG0000010000", n=500, pheno_var=1.5) + assert "CELL" in h.chunk_pl.columns + assert "GENE" in h.chunk_pl.columns + assert h.chunk_pl["CELL"].unique().to_list() == ["T_gd"] + assert h.chunk_pl["GENE"].unique().to_list() == ["ENSG0000010000"] + + def test_p_value_column_present(self, mapping_file, sample_qtl_df): + h = Harmonize(mapping_file, "dummy", "qtl", None, "quant", None, None, False) + h.create_mapping() + h.harmonize(sumstat=sample_qtl_df, cell="T_gd", gene="ENSG0000010000", n=500, pheno_var=1.5) + assert "P" in h.chunk_pl.columns + assert all(0 <= p <= 1 for p in h.chunk_pl["P"].to_list()) + + def test_missing_n_raises(self, mapping_file, sample_qtl_df): + h = Harmonize(mapping_file, "dummy", "qtl", None, "quant", None, None, False) + h.create_mapping() + with pytest.raises(HarmonizationError): + h.harmonize(sumstat=sample_qtl_df.drop("N"), cell="T_gd", + gene="ENSG0000010000", pheno_var=1.5) + + def test_maf_filter_reduces_rows(self, mapping_file, sample_qtl_df): + """All rows in sample have EAF=0 (default), so MAF=0; maf=0.1 should drop all.""" + h = Harmonize(mapping_file, "dummy", "qtl", None, "quant", None, maf=0.1, permuted=False) + h.create_mapping() + h.harmonize(sumstat=sample_qtl_df, cell="T_gd", gene="ENSG0000010000", n=500, pheno_var=1.5) + # With maf=0.1 all rows may be filtered out (EAF defaults to 0) + assert h.chunk_pl.height >= 0 # just confirm no crash + + def test_binary_trait_n_columns(self, mapping_file, sample_qtl_df): + h = Harmonize(mapping_file, "dummy", "gwas", None, "binary", None, None, False) + h.create_mapping() + gwas_df = pl.DataFrame({ + "Chr": [1, 1], + "pos": [100, 200], + "a0": ["A", "C"], + "a1": ["T", "G"], + "p": [0.01, 0.5], + "beta": [0.1, -0.2], + "se": [0.05, 0.08], + "N": [500, 500], + "Gene": ["T1", "T1"], + "cell.type": ["gwas", "gwas"], + }) + h.harmonize(sumstat=gwas_df, trait="T1", n_cases=200, n_controls=300, pheno_var=1.0) + assert "N_CASES" in h.chunk_pl.columns + assert "N_CONTROLS" in h.chunk_pl.columns + + def test_invalid_type_trait_raises(self, mapping_file, sample_qtl_df): + h = Harmonize(mapping_file, "dummy", "qtl", None, "invalid_type", None, None, False) + h.create_mapping() + with pytest.raises(HarmonizationError, match="binary or quant"): + h.harmonize(sumstat=sample_qtl_df, cell="T_gd", gene="ENSG0000010000", n=500, pheno_var=1.5) + + def test_empty_dataframe_harmonizes_without_crash(self, mapping_file, empty_qtl_df): + """Harmonizing an empty DataFrame should not raise and should produce an empty chunk_pl.""" + h = Harmonize(mapping_file, "dummy", "qtl", None, "quant", None, None, False) + h.create_mapping() + h.harmonize(sumstat=empty_qtl_df, cell="T_gd", gene="ENSG0000010000", n=500, pheno_var=1.5) + assert h.chunk_pl.height == 0 + + +# --------------------------------------------------------------------------- +# WriterMixin +# --------------------------------------------------------------------------- + +class TestWriterMixin: + def test_ingest_appends_rows(self, mapping_file, sample_qtl_df, tmp_path): + uri = str(tmp_path / "writer_test") + h = Harmonize(mapping_file, uri, "qtl", None, "quant", None, None, False) + h.create_tiledb() + h.create_mapping() + h.harmonize(sumstat=sample_qtl_df, cell="T_gd", gene="ENSG0000010000", n=500, pheno_var=1.5) + h.ingest_data(file_path="dummy_file.tsv") + + with tiledb.open(uri, mode="r") as A: + df = A.query(dims=["CHR", "CELL", "GENE", "POS"]).df[:] + assert len(df) > 0 + + def test_duplicates_are_removed(self, mapping_file, tmp_path): + """Rows sharing an index key (CHR, POS, CELL, GENE) are ALL dropped. + + The ingestion pipeline treats ambiguous duplicate positions as + unreliable and excludes them entirely rather than keeping one. + """ + uri = str(tmp_path / "dedup_test") + h = Harmonize(mapping_file, uri, "qtl", None, "quant", None, None, False) + h.create_tiledb() + h.create_mapping() + + # Two identical rows + dup_df = pl.DataFrame({ + "Chr": [20, 20], + "Gene": ["ENSG0000010000", "ENSG0000010000"], + "cell.type": ["T_gd", "T_gd"], + "pos": [500, 500], # same position → duplicate + "a0": ["A", "A"], + "a1": ["T", "T"], + "p": [0.01, 0.01], + "N": [500, 500], + "beta": [0.1, 0.1], + "se": [0.05, 0.05], + }) + h.harmonize(sumstat=dup_df, cell="T_gd", gene="ENSG0000010000", n=500, pheno_var=1.5) + h.ingest_data(file_path="dup_file.tsv") + + with tiledb.open(uri, mode="r") as A: + df = A.query(dims=["CHR", "CELL", "GENE", "POS"]).df[:] + # Both rows are duplicate; they should both be dropped by the dedup logic + assert len(df) == 0 + + +# --------------------------------------------------------------------------- +# MetadataMixin +# --------------------------------------------------------------------------- + +class TestMetadataMixin: + def _create_ingested_tiledb(self, mapping_file, sample_qtl_df, tmp_path): + """Helper: create + ingest + return a Harmonize object with data.""" + uri = str(tmp_path / "meta_test") + h = Harmonize(mapping_file, uri, "qtl", None, "quant", None, None, False) + h.create_tiledb() + h.create_mapping() + h.harmonize(sumstat=sample_qtl_df, cell="T_gd", gene="ENSG0000010000", n=500, pheno_var=1.5) + h.ingest_data(file_path="file.tsv") + return h + + def test_create_metadata_writes_json(self, mapping_file, sample_qtl_df, tmp_path): + h = self._create_ingested_tiledb(mapping_file, sample_qtl_df, tmp_path) + h.create_metadata(file_path="file.tsv") + + metadata_dir = h.uri + "_metadata_parts" + json_files = list(os.listdir(metadata_dir)) + assert len(json_files) == 1 + assert json_files[0].endswith(".json") + + def test_metadata_json_has_cell_key(self, mapping_file, sample_qtl_df, tmp_path): + h = self._create_ingested_tiledb(mapping_file, sample_qtl_df, tmp_path) + h.create_metadata(file_path="file.tsv") + + metadata_dir = h.uri + "_metadata_parts" + json_file = os.path.join(metadata_dir, os.listdir(metadata_dir)[0]) + with open(json_file) as f: + meta = json.load(f) + assert "CELL" in meta + assert "T_gd" in meta["CELL"] + + def test_merge_metadata_stores_in_tiledb(self, mapping_file, sample_qtl_df, tmp_path): + h = self._create_ingested_tiledb(mapping_file, sample_qtl_df, tmp_path) + h.create_metadata(file_path="file.tsv") + h.merge_metadata_files() + + with tiledb.open(h.uri, mode="r") as A: + merged = json.loads(A.meta["merged_metadata"]) + assert "CELL" in merged + assert "T_gd" in merged["CELL"] + + def test_merge_metadata_static_helper(self): + """_merge_single_metadata should correctly merge QTL entries.""" + merged = {"traits": [], "CELL": []} + file_meta = { + "CELL": ["T_gd"], + "T_gd": { + "20": {"ENSG0000010000": {"ACAT": 0.5, "N": 500.0, "PHENO_VAR": 1.5, "MIN_P": 0.001}} + }, + } + MetadataMixin._merge_single_metadata(merged, file_meta) + assert "T_gd" in merged["CELL"] + assert "20" in merged["T_gd"] + assert "ENSG0000010000" in merged["T_gd"]["20"] + + def test_export_metadata_to_csv(self, mapping_file, sample_qtl_df, tmp_path): + h = self._create_ingested_tiledb(mapping_file, sample_qtl_df, tmp_path) + h.create_metadata(file_path="file.tsv") + h.merge_metadata_files() + + import pandas as pd + df = h.export_metadata_to_csv() + assert isinstance(df, pd.DataFrame) + assert len(df) > 0 + assert "CELL" in df.columns + + +# --------------------------------------------------------------------------- +# End-to-end ingestion using example data files +# --------------------------------------------------------------------------- + +class TestEndToEndIngestion: + def test_full_qtl_pipeline(self, mapping_file, tmp_path): + """Full QTL ingestion pipeline using example data files.""" + uri = str(tmp_path / "e2e_qtl") + h = Harmonize(mapping_file, uri, "qtl", None, "quant", None, None, False) + h.create_tiledb() + h.create_mapping() + + data_files = [ + (os.path.join(EXAMPLE_DATA_DIR, "dummy_out_ENSG0000010000.tsv.gz"), "ENSG0000010000"), + (os.path.join(EXAMPLE_DATA_DIR, "dummy_out_ENSG0000010001.tsv.gz"), "ENSG0000010001"), + ] + + for filepath, gene in data_files: + chunk_pl = pl.read_csv(filepath, separator="\t", low_memory=True, null_values="NA") + h.harmonize(sumstat=chunk_pl, cell="Tgd", gene=gene, n=4000, pheno_var=1.5) + h.ingest_data(file_path=filepath) + h.create_metadata(file_path=filepath) + + h.merge_metadata_files() + + # Verify data was written + with tiledb.open(uri, mode="r") as A: + df = A.query(dims=["CHR", "CELL", "GENE", "POS"]).df[:] + assert len(df) > 0 + + # Verify merged metadata + with tiledb.open(uri, mode="r") as A: + merged = json.loads(A.meta["merged_metadata"]) + assert "Tgd" in merged["CELL"] + + def test_backward_compat_import(self): + """harmonize_ingest shim must re-export the same Harmonize class.""" + from tdbsumstat.utils.harmonize_ingest import Harmonize as OldHarmonize + from tdbsumstat.utils.ingest import Harmonize as NewHarmonize + assert OldHarmonize is NewHarmonize + + +# --------------------------------------------------------------------------- +# Module smoke-tests +# --------------------------------------------------------------------------- + +class TestModuleImports: + def test_import_errors(self): + from tdbsumstat.utils.ingest.errors import HarmonizationError # noqa: F401 + + def test_import_schema(self): + from tdbsumstat.utils.ingest.schema import SchemaMixin # noqa: F401 + + def test_import_mapping(self): + from tdbsumstat.utils.ingest.mapping import MappingMixin # noqa: F401 + + def test_import_harmonize(self): + from tdbsumstat.utils.ingest.harmonize import HarmonizeMixin # noqa: F401 + + def test_import_qc(self): + from tdbsumstat.utils.ingest.qc import QCMixin # noqa: F401 + + def test_import_writer(self): + from tdbsumstat.utils.ingest.writer import WriterMixin # noqa: F401 + + def test_import_metadata(self): + from tdbsumstat.utils.ingest.metadata import MetadataMixin # noqa: F401 + + def test_import_package(self): + from tdbsumstat.utils.ingest import Harmonize, HarmonizationError # noqa: F401 From a77d8101b7f15fe0cd754bad9f02597c8d15bf91 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 5 Mar 2026 19:57:25 +0000 Subject: [PATCH 4/6] Nextflow: fix pipeline bugs, add stub blocks, CI workflow, and update docs Co-authored-by: bruno-ariano <26384813+bruno-ariano@users.noreply.github.com> --- .github/workflows/ci.yml | 163 +++++++++++ README.md | 54 ++++ conf/test.config | 71 ++--- docs/README.md | 302 ++++++++++++++++----- example_data/example_data_table_test.csv | 3 + example_data/recompute_meta_test_table.csv | 2 + example_data/trait_list_test.csv | 3 + main.nf | 108 ++++---- modules/create_tiledb/main.nf | 9 +- modules/ingestion/main.nf | 34 ++- modules/locusbreaker/main.nf | 29 +- modules/merge_metadata/main.nf | 115 ++++---- modules/recompute_meta/main.nf | 40 +-- modules/regions/main.nf | 42 +-- modules/snp/main.nf | 11 +- modules/traits/main.nf | 36 ++- 16 files changed, 744 insertions(+), 278 deletions(-) create mode 100644 .github/workflows/ci.yml create mode 100644 example_data/example_data_table_test.csv create mode 100644 example_data/recompute_meta_test_table.csv create mode 100644 example_data/trait_list_test.csv diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..f563f7f --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,163 @@ +name: CI + +on: + push: + branches: + - main + - master + - develop + pull_request: + branches: + - main + - master + - develop + +# Minimal permissions for all jobs +permissions: + contents: read + +jobs: + # ----------------------------------------------------------------------- + # Python unit + integration tests + # ----------------------------------------------------------------------- + python-tests: + name: Python tests (pytest) + runs-on: ubuntu-latest + permissions: + contents: read + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python 3.11 + uses: actions/setup-python@v5 + with: + python-version: "3.11" + cache: "pip" + + - name: Install tdbsumstat and test dependencies + run: | + pip install -e . + pip install pytest pytest-cov + + - name: Run test suite + run: python -m pytest tests/ -v --tb=short + + # ----------------------------------------------------------------------- + # Nextflow stub tests (validate workflow DAG without running commands) + # ----------------------------------------------------------------------- + nextflow-stub-tests: + name: Nextflow stub tests + runs-on: ubuntu-latest + permissions: + contents: read + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Install Nextflow + uses: nf-core/setup-nextflow@v2 + with: + version: "latest-stable" + + - name: Stub test – ingestion + run: | + nextflow run main.nf -profile test_ingest -stub -ansi-log false + echo "✓ Ingestion stub test passed" + + - name: Stub test – export SNP + run: | + nextflow run main.nf -profile test_export_snp -stub -ansi-log false + echo "✓ Export SNP stub test passed" + + - name: Stub test – export locusbreaker + run: | + nextflow run main.nf -profile test_export_lb -stub -ansi-log false + echo "✓ Export locusbreaker stub test passed" + + - name: Stub test – export traits + run: | + nextflow run main.nf -profile test_export_traits -stub -ansi-log false + echo "✓ Export traits stub test passed" + + - name: Stub test – export regions + run: | + nextflow run main.nf -profile test_export_regions -stub -ansi-log false + echo "✓ Export regions stub test passed" + + - name: Stub test – recompute metadata + run: | + nextflow run main.nf -profile test_recompute_meta -stub -ansi-log false + echo "✓ Recompute metadata stub test passed" + + # ----------------------------------------------------------------------- + # Full Nextflow integration test (runs tdbsumstat commands via local Python) + # ----------------------------------------------------------------------- + nextflow-integration-test: + name: Nextflow integration test (ingest + export) + runs-on: ubuntu-latest + permissions: + contents: read + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python 3.11 + uses: actions/setup-python@v5 + with: + python-version: "3.11" + cache: "pip" + + - name: Install tdbsumstat + run: pip install -e . + + - name: Install Nextflow + uses: nf-core/setup-nextflow@v2 + with: + version: "latest-stable" + + - name: Create CI data table with absolute file paths + run: | + { + echo "FILE,CELL,GENE,PHENO_VAR,N" + echo "$(pwd)/example_data/dummy_out_ENSG0000010000.tsv.gz,Tgd,ENSG0000010000,1.5,4000" + echo "$(pwd)/example_data/dummy_out_ENSG0000010001.tsv.gz,Tgd,ENSG0000010001,1.5,4000" + } > /tmp/example_data_table_ci.csv + + - name: Run ingestion pipeline (local, no container) + run: | + nextflow run main.nf \ + --ingestion true \ + --file_path_ingestion /tmp/example_data_table_ci.csv \ + --mapping_file "$(pwd)/example_data/mapping_file_test.csv" \ + --type_sumstat qtl \ + --tiledb_name test_ci \ + --ingestion_chunk_files 2 \ + --maf 0 \ + --mac 0 \ + --outdir ./results_ci \ + -process.container null \ + -ansi-log false + + - name: Verify TileDB was created + run: | + ls -la results_ci/TileDB/TileDB_test_ci/ + echo "✓ TileDB created successfully" + + - name: Run SNP export (local, no container) + run: | + nextflow run main.nf \ + --export true \ + --snp "$(pwd)/example_data/snp_list_sc.csv" \ + --tiledb_path "$(pwd)/results_ci/TileDB/TileDB_test_ci" \ + --uri_path "$(pwd)/results_ci/TileDB/TileDB_test_ci" \ + --attrs "SE,BETA" \ + --out out_ci \ + --type_sumstat qtl \ + --outdir ./results_ci \ + -process.container null \ + -ansi-log false + echo "✓ SNP export completed" diff --git a/README.md b/README.md index d1ed1e6..9ee03a0 100755 --- a/README.md +++ b/README.md @@ -158,3 +158,57 @@ Test files: | `tests/test_ingest.py` | Each ingest mixin + end-to-end pipeline with example data | | `tests/test_export.py` | All export modules + CLI command routing | +--- + +### Testing the Nextflow pipeline + +#### Stub tests (no data required) + +Every Nextflow module has a `stub` block that creates placeholder output files +instead of running the actual `tdbsumstat` commands. Stub tests validate the +workflow DAG structure (channels, process I/O, publish dirs) without needing +containers or real data. + +```bash +# Install Nextflow first: https://www.nextflow.io/docs/latest/install.html + +nextflow run main.nf -profile test_ingest -stub # ingestion DAG +nextflow run main.nf -profile test_export_snp -stub # SNP export DAG +nextflow run main.nf -profile test_export_regions -stub # region export DAG +nextflow run main.nf -profile test_export_traits -stub # trait export DAG +nextflow run main.nf -profile test_export_lb -stub # locusbreaker DAG +nextflow run main.nf -profile test_recompute_meta -stub # recompute metadata DAG +``` + +These stub tests are run automatically on every push and pull request via +[`.github/workflows/ci.yml`](.github/workflows/ci.yml). + +#### Full integration test (local Python + Nextflow) + +```bash +# 1. Install tdbsumstat +pip install -e . + +# 2. Build a data table with absolute paths +{ + echo "FILE,CELL,GENE,PHENO_VAR,N" + echo "$(pwd)/example_data/dummy_out_ENSG0000010000.tsv.gz,Tgd,ENSG0000010000,1.5,4000" + echo "$(pwd)/example_data/dummy_out_ENSG0000010001.tsv.gz,Tgd,ENSG0000010001,1.5,4000" +} > /tmp/example_data_table_ci.csv + +# 3. Run ingestion (uses local Python – no container needed) +nextflow run main.nf \ + --ingestion true \ + --file_path_ingestion /tmp/example_data_table_ci.csv \ + --mapping_file "$(pwd)/example_data/mapping_file_test.csv" \ + --type_sumstat qtl \ + --tiledb_name test_ci \ + --ingestion_chunk_files 2 \ + --maf 0 --mac 0 \ + --outdir ./results_ci \ + -process.container null \ + -ansi-log false +``` + +See [`docs/README.md`](docs/README.md) for the full Nextflow parameter reference. + diff --git a/conf/test.config b/conf/test.config index 272edb2..3caa0ae 100644 --- a/conf/test.config +++ b/conf/test.config @@ -2,25 +2,24 @@ profiles { test_ingest { params { - ingest = true + ingest = true + ingestion = true file_path_ingestion = "${projectDir}/example_data/example_data_table.csv" mapping_file = "${projectDir}/example_data/mapping_file_test.csv" - ingestion = true type_sumstat = "qtl" qc = false ingestion_chunk_files = 2 tiledb_name = "test" - outdir = "./results" // Add this missing parameter - publish_dir_mode = 'copy' // Add this missing parameter + outdir = "./results" + publish_dir_mode = 'copy' } } test_export_lb { params { - export = true - //uri_path = "/lustre/scratch124/humgen/projects_v2/cardinal_analysis/analysis/core_dataset/freeze3/tiledbs/Tensor/TileDB_metanalyses_gh_celltype2_freeze3" - uri_path = "${projectDir}/example_data/TileDB_test" + export = true locusbreaker = true + uri_path = "${projectDir}/example_data/TileDB_test" out = "out_test" type_sumstat = "qtl" hole_lb = 250000 @@ -28,72 +27,76 @@ profiles { maf_lb = 0.001 locus_max_size_lb = 3000000 cis_trans_lb = "cis" - table_lb = "${projectDir}/example_data/locusbreaker_test_table_sc.csv" - batch_name = 1 - outdir = "./results" // Add this missing parameter - publish_dir_mode = 'copy' // Add this missing parameter + table_lb = "${projectDir}/example_data/locusbreaker_test_table_sc.csv" + outdir = "./results" + publish_dir_mode = 'copy' } } + test_export_snp { params { - export = true - uri_path = "${projectDir}/example_data/TileDB_test" + export = true snp = "${projectDir}/example_data/snp_list_sc.csv" + tiledb_path = "${projectDir}/example_data/TileDB_test" + uri_path = "${projectDir}/example_data/TileDB_test" attrs = "SE,BETA" out = "out" type_sumstat = "qtl" - outdir = "./results" // Add this missing parameter - publish_dir_mode = 'copy' // Add this missing parameter + outdir = "./results" + publish_dir_mode = 'copy' } } + test_export_regions { params { - export = true - uri_path = "${projectDir}/example_data/TileDB_test" + export = true regions = "${projectDir}/example_data/region_list_sc.csv" + uri_path = "${projectDir}/example_data/TileDB_test" attrs = "SE,BETA" out = "out" type_sumstat = "qtl" - outdir = "./results" // Add this missing parameter - publish_dir_mode = 'copy' // Add this missing parameter + tiledb_batch_size = 1 + outdir = "./results" + publish_dir_mode = 'copy' } } + test_export_traits { params { - export = true + export = true export_traits = true uri_path = "${projectDir}/example_data/TileDB_test" - list_traits = "${projectDir}/example_data/trait_list.csv" + list_traits = "${projectDir}/example_data/trait_list_test.csv" out = "out" type_sumstat = "qtl" - outdir = "./results" // Add this missing parameter - publish_dir_mode = 'copy' // Add this missing parameter + outdir = "./results" + publish_dir_mode = 'copy' tiledb_batch_size = 1 } } + test_export_meta { params { - export = true + export = true export_meta = true uri_path = "${projectDir}/example_data/TileDB_test" - list_traits = "${projectDir}/example_data/trait_list.csv" out = "out" type_sumstat = "qtl" - outdir = "./results" // Add this missing parameter - publish_dir_mode = 'copy' // Add this missing parameter - tiledb_batch_size = 100 + outdir = "./results" + publish_dir_mode = 'copy' } } + test_recompute_meta { params { - recompute_meta = true - uri_path = "/project/cardinal/QTLs/freeze3/TileDBs/gh_metanalyses/TileDB_metanalyses_gh_celltype2_freeze3" - list_traits = "${projectDir}/gh_meta.csv" - mac =10 + recompute_meta = true + uri_path = "${projectDir}/example_data/TileDB_test" + list_traits = "${projectDir}/example_data/recompute_meta_test_table.csv" + mac = 10 out = "out" type_sumstat = "qtl" - outdir = "./results" // Add this missing parameter - publish_dir_mode = 'copy' // Add this missing parameter + outdir = "./results" + publish_dir_mode = 'copy' tiledb_batch_size = 10 } } diff --git a/docs/README.md b/docs/README.md index 0f88c9b..89db0d4 100755 --- a/docs/README.md +++ b/docs/README.md @@ -11,7 +11,14 @@ TileDB-sumstat is a Nextflow + Python toolkit for scalable ingestion and export - [Export](#export) - [SNP-based Export](#snp-based-export) - [Region-based Export](#region-based-export) + - [Traits Export](#traits-export) - [Locusbreaker](#locusbreaker) + - [Metadata Export](#metadata-export) + - [Recompute Metadata](#recompute-metadata) +- [Nextflow Modules](#nextflow-modules) +- [Testing the Nextflow Pipeline](#testing-the-nextflow-pipeline) + - [Stub Tests (CI / no data required)](#stub-tests-ci--no-data-required) + - [Integration Test](#integration-test) - [Test Data](#test-data) - [Support and Contribution](#support-and-contribution) @@ -41,7 +48,7 @@ TileDB-sumstat implements two main workflows executed in separate steps: 2. **Export** — Query the TileDB array and export results by: - SNP - Genomic region - - Entire summary statistics + - Entire summary statistics (trait-based) - Clumping using the "Locusbreaker" algorithm The program can be used with Nextflow or as standalone Python utilities. @@ -58,36 +65,48 @@ Import summary statistics files into a TileDB array. #### Required Files -- **Mapping File** (.csv): Maps input GWAS columns to standard TileDB-sumstat columns. See `example_data/mapping_file_test` for format. First column: original GWAS names, second column: converted names. -- **Data Table**: Lists files to ingest. See `example_data/example_data_table.csv`. Must include: - - Path to GWAS files - - Optional: Additional columns can be added to this file if they are not present in the summary statistics: - - `N`: Sample size for specific summary statistics - - `N_CASES`: Sample size for cases specific to a summary statistics - - `N_CONTROLS`: Sample size for controls specific to a summary statistics - - `CELL`: Cell name for single QTL studies - - `GENE`: GENE name for single QTL studies - - `PHENO_VAR`: Phenotypic variance for the specific trait (only for QTLs single cell) - - `TRAIT`: Trait name for GWAS studies +- **Mapping File** (.csv): Maps input GWAS columns to standard TileDB-sumstat columns. See `example_data/mapping_file_test.csv` for format. First column: original GWAS names, second column: converted names. +- **Data Table** (`example_data/example_data_table.csv`): Lists files to ingest. Must contain **absolute file paths** in the `FILE` column. Optional columns: + - `N`: Sample size + - `N_CASES` / `N_CONTROLS`: Case/control sizes for binary GWAS + - `CELL`: Cell type (single-cell QTL) + - `GENE`: Gene name (single-cell QTL) + - `PHENO_VAR`: Phenotypic variance (single-cell QTL) + - `TRAIT`: Trait name (GWAS) + +> **Note on file paths:** The `FILE` column in the data table must contain absolute paths that are accessible from the compute nodes running the pipeline. The file `example_data/example_data_table_test.csv` shows the column format; for production runs, replace the paths with absolute paths on your cluster. #### Parameters **Required:** -- `--ingest` (flag to enable ingestion) +- `--ingestion` (flag to enable ingestion) - `--file_path_ingestion` (path to data table file) +- `--mapping_file` (path to column mapping file) - `--type_sumstat` (either `gwas` or `qtl`) +- `--tiledb_name` (name for the TileDB array) **Optional:** -- `--qc` (enable QC processing) -- `--ingestion_chunk_files` (number of files to ingest simultaneously, default: 4) +- `--qc` (enable QC processing via gwaslab) +- `--ingestion_chunk_files` (number of files to ingest per batch, default: 4) +- `--maf` (minimum allele frequency filter, default: 0) +- `--mac` (minimum allele count filter, default: 0) +- `--permuted` (compute SE from permuted p-value) +- `--pvar_file` (pvar file to align alleles) +- `--outdir` (output directory, default: `./results`) #### Example ```bash -nextflow run main.nf -profile singularity --ingest --file_path_ingestion example_data/example_data_table.csv --mapping_file example_data/mapping_file_test.csv --type_sumstat qtl --ingestion_chunk_files 4 +nextflow run main.nf \ + -profile singularity \ + --ingestion \ + --file_path_ingestion example_data/example_data_table.csv \ + --mapping_file example_data/mapping_file_test.csv \ + --type_sumstat qtl \ + --tiledb_name my_tiledb \ + --ingestion_chunk_files 4 ``` -Using Nextflow profile you can also use - +Using a pre-defined test profile: ```bash nextflow run main.nf -profile test_ingest,singularity ``` @@ -99,85 +118,246 @@ Query and export data from the TileDB array. #### Common Parameters -**Required:**: -- --export (flag to enable export) -- --tiledb_path (path to TileDB array) -- --out (output file prefix) -- --type_sumstat (type of summary statistics, either gwas or qtl for single cell) +**Required:** +- `--export` (flag to enable export) +- `--uri_path` (path to TileDB array) +- `--out` (output file prefix) +- `--type_sumstat` (type of summary statistics: `gwas` or `qtl`) -**Optional**: -- --attrs (attributes to export, e.g., "BETA,SE,PVAL,EAF,A1,A2") +**Optional:** +- `--attrs` (attributes to export, e.g., `"BETA,SE,EAF"`, default: `"P,SNPID,EAF,BETA,SE"`) #### SNP-based Export Extract specific SNP positions. -**Required** -- --snp (path to SNP list file) +**Required:** +- `--snp` (path to SNP list CSV, columns: `CHR`, `POS`, `TRAIT`) + +Example files: `example_data/snp_list_sc.csv` -Example Files: -- GWAS: example_data/snp_list_gwas.csv -- Single-cell: example_data/snp_list_sc.csv +```bash +nextflow run main.nf \ + -profile singularity \ + --export \ + --snp example_data/snp_list_sc.csv \ + --tiledb_path /path/to/tiledb \ + --uri_path /path/to/tiledb \ + --attrs "BETA,SE,P" \ + --out results/snp_output \ + --type_sumstat qtl +``` -#### Example: +Quick test (stub – no data needed): ```bash -nextflow run main.nf -profile singularity --export --tiledb_path /path/to/tiledb --snp /path/to/snp_list.csv --attrs "BETA,SE,PVAL,EAF,A1,A2" --out /path/to/output_prefix --type_sumstat gwas +nextflow run main.nf -profile test_export_snp -stub ``` + #### Region-based Export -Extract genomic regions using BED format. +Extract genomic intervals. **Required:** -- --table-regions (path to regions file) +- `--regions` (path to regions CSV, columns: `CHR`, `START`, `END`, `TRAIT`) + +Example file: `example_data/region_list_sc.csv` -Example: ```bash -nextflow run main.nf -profile singularity --export --tiledb_path /path/to/tiledb --table-regions /path/to/regions_table.csv --attrs "BETA,SE,PVAL" --out /path/to/output_prefix --type_sumstat gwas +nextflow run main.nf \ + -profile singularity \ + --export \ + --regions example_data/region_list_sc.csv \ + --uri_path /path/to/tiledb \ + --attrs "BETA,SE,P" \ + --out results/regions_output \ + --type_sumstat qtl ``` -Quick Test: +Quick test: ```bash -nextflow run main.nf -profile test_export_lb,singularity +nextflow run main.nf -profile test_export_regions -stub +``` + +#### Traits Export + +Export complete summary statistics for a list of traits or cell-type/gene combinations. + +**Required:** +- `--export_traits` (flag) +- `--list_traits` (path to CSV with `TRAIT` column; for QTL: `CELL:GENE` format) + +Example file: `example_data/trait_list_test.csv` + +```bash +nextflow run main.nf \ + -profile singularity \ + --export \ + --export_traits \ + --list_traits example_data/trait_list_test.csv \ + --uri_path /path/to/tiledb \ + --out results/traits_output \ + --type_sumstat qtl +``` + +Quick test: +```bash +nextflow run main.nf -profile test_export_traits -stub ``` #### Locusbreaker -Identify genomic loci with significant associations and export locus-centric results. +Identify genomic loci with significant associations. + +**Required:** +- `--locusbreaker` (flag) +- `--table_lb` (path to traits table, columns: `CHR`, `TRAIT`, `SIG`, `LIM`) + +**Optional:** +- `--maf_lb` (MAF filter, default: 0.001) +- `--locus_max_size_lb` (maximum locus size in bp, default: 3 Mb) +- `--hole_lb` (maximum gap within loci in bp, default: 250 kb) +- `--cis_trans_lb` (for QTLs: `cis` or `trans`, default: `cis`) + +Example: +```bash +nextflow run main.nf \ + -profile singularity \ + --locusbreaker \ + --table_lb example_data/locusbreaker_test_table_sc.csv \ + --uri_path /path/to/tiledb \ + --out results/lb_output \ + --type_sumstat qtl +``` + +Quick test: +```bash +nextflow run main.nf -profile test_export_lb -stub +``` + +#### Metadata Export + +Export the merged metadata stored in the TileDB array to CSV. + +```bash +tdbsumstat export --export-meta --uri-path /path/to/tiledb --type-sumstat qtl --out metadata_out +``` + +#### Recompute Metadata + +Recompute per-trait/cell metadata statistics after applying a MAC filter without modifying the TileDB data. **Required:** -- --table-lb (path to traits table, see example_data/locusbreaker_test_table.csv) +- `--recompute_meta` (flag) +- `--list_traits` (CSV with `CELL` and `CHR` columns for QTL, or `TRAIT` and `CHR` for GWAS) +- `--uri_path` +- `--mac` (minimum allele count threshold) + +Example file: `example_data/recompute_meta_test_table.csv` -**Optional** -- --maf-lb (minor allele frequency filter) -- --locus-max-size (maximum locus size in base pairs) -- --hole-lb (maximum gap size within loci in base pairs) -- --cis-trans (for QTLs: filter by cis or trans) +```bash +nextflow run main.nf \ + -profile singularity \ + --recompute_meta \ + --list_traits example_data/recompute_meta_test_table.csv \ + --uri_path /path/to/tiledb \ + --mac 10 \ + --out results/meta_out \ + --type_sumstat qtl +``` -Locusbreaker Algorithm: -1. Select SNPs below p-value threshold (suggested: 1e-6) -2. Group consecutive SNPs within distance threshold (suggested: 250 kb) -3. Retain groups containing at least one genome-wide significant SNP (suggested: 5e-8) -4. Expand locus boundaries by margin (e.g., +100 kb) -5. Apply additional filters (MAF, locus size, cis/trans) +Quick test: +```bash +nextflow run main.nf -profile test_recompute_meta -stub +``` --- -#### Metadata extraction +## Nextflow Modules + +The pipeline is composed of the following Nextflow process modules under `modules/`: + +| Module | Process | Description | +|--------|---------|-------------| +| `create_tiledb/` | `CREATE_TILEDB` | Creates the TileDB sparse array schema | +| `ingestion/` | `INGEST_DATA` | Harmonises and ingests one file-list chunk into TileDB | +| `merge_metadata/` | `MERGE_METADATA` | Collects per-chunk metadata JSON files and stores merged metadata in TileDB | +| `snp/` | `EXPORT_SNP` | Exports data for a list of SNP positions | +| `regions/` | `EXPORT_REGIONS` | Exports data for a list of genomic intervals | +| `traits/` | `TRAITS` | Exports complete summary statistics for a list of traits | +| `locusbreaker/` | `EXPORT_LOCUSBREAKER` | Runs the Locusbreaker algorithm and exports locus/segment tables | +| `recompute_meta/` | `RECOMPUTE_META` | Recomputes metadata with a MAC filter | -Metadata are structured as json in tiledb. To extract them you can use the following command: +Each module has a `stub` block so the workflow DAG can be validated without executing the actual commands (see [Testing](#testing-the-nextflow-pipeline)). + +--- + +## Testing the Nextflow Pipeline + +### Stub Tests (CI / no data required) + +Stub tests validate the workflow DAG structure (channels, processes, outputs) without running the actual `tdbsumstat` commands. They are the recommended way to test the pipeline in CI or when data is not available. + +All test profiles are defined in `conf/test.config` and registered in `nextflow.config`. ```bash -tdbsumstat export metadata +# Validate ingestion workflow +nextflow run main.nf -profile test_ingest -stub + +# Validate export workflows +nextflow run main.nf -profile test_export_snp -stub +nextflow run main.nf -profile test_export_regions -stub +nextflow run main.nf -profile test_export_traits -stub +nextflow run main.nf -profile test_export_lb -stub +nextflow run main.nf -profile test_recompute_meta -stub ``` -tiledb_meta -{'traits': [], 'CELL': ['Tgd'], 'Tgd': {'20': {'ENSG0000010000': {'ACAT': 0.0, 'N': 4000.0, 'PHENO_VAR': 1.0}, 'ENSG0000010001': {'ACAT': 0.0, 'N': 4000.0, 'PHENO_VAR': 1.0}}}} +These stubs are also run automatically on every push and pull request via the [CI workflow](.github/workflows/ci.yml). + +### Integration Test + +To run a full integration test that actually executes the ingestion and export commands, you need: +1. `tdbsumstat` installed (`pip install -e .`) +2. Nextflow installed + +```bash +# Build a CI-friendly data table with absolute paths +{ + echo "FILE,CELL,GENE,PHENO_VAR,N" + echo "$(pwd)/example_data/dummy_out_ENSG0000010000.tsv.gz,Tgd,ENSG0000010000,1.5,4000" + echo "$(pwd)/example_data/dummy_out_ENSG0000010001.tsv.gz,Tgd,ENSG0000010001,1.5,4000" +} > /tmp/example_data_table_ci.csv + +# Run ingestion (no container – uses local tdbsumstat) +nextflow run main.nf \ + --ingestion true \ + --file_path_ingestion /tmp/example_data_table_ci.csv \ + --mapping_file "$(pwd)/example_data/mapping_file_test.csv" \ + --type_sumstat qtl \ + --tiledb_name test_ci \ + --ingestion_chunk_files 2 \ + --maf 0 \ + --mac 0 \ + --outdir ./results_ci \ + -process.container null \ + -ansi-log false +``` + +--- + ## Test Data -- example_data/snp_list.csv - Example SNP list for SNP-based export -- example_data/example_data_table.csv - Example table for regions and ingestion -- example_data/mapping_file_test - Example mapping file for ingestion -- example_data/locusbreaker_test_table.csv - Example table for Locusbreaker +| File | Description | +|------|-------------| +| `example_data/dummy_out_ENSG0000010000.tsv.gz` | Example QTL summary statistics (gene ENSG0000010000) | +| `example_data/dummy_out_ENSG0000010001.tsv.gz` | Example QTL summary statistics (gene ENSG0000010001) | +| `example_data/example_data_table.csv` | Ingestion data table (absolute paths, for cluster use) | +| `example_data/example_data_table_test.csv` | Ingestion data table (filenames only, documents column format) | +| `example_data/mapping_file_test.csv` | Column mapping file for example data | +| `example_data/snp_list_sc.csv` | SNP list for SNP-based export tests | +| `example_data/region_list_sc.csv` | Region list for region-based export tests | +| `example_data/locusbreaker_test_table_sc.csv` | Traits table for Locusbreaker tests | +| `example_data/trait_list_test.csv` | Trait list (`TRAIT` column) for traits export tests | +| `example_data/recompute_meta_test_table.csv` | Cell/CHR table for recompute metadata tests | --- @@ -190,5 +370,3 @@ Contributions are welcome! Please: 2. Create a feature branch 3. Submit a pull request against the main branch 4. Follow the repository's contribution guidelines - ---- \ No newline at end of file diff --git a/example_data/example_data_table_test.csv b/example_data/example_data_table_test.csv new file mode 100644 index 0000000..1126c56 --- /dev/null +++ b/example_data/example_data_table_test.csv @@ -0,0 +1,3 @@ +FILE,CELL,GENE,PHENO_VAR,N +dummy_out_ENSG0000010000.tsv.gz,Tgd,ENSG0000010000,1.5,4000 +dummy_out_ENSG0000010001.tsv.gz,Tgd,ENSG0000010001,1.5,4000 \ No newline at end of file diff --git a/example_data/recompute_meta_test_table.csv b/example_data/recompute_meta_test_table.csv new file mode 100644 index 0000000..9bce1f7 --- /dev/null +++ b/example_data/recompute_meta_test_table.csv @@ -0,0 +1,2 @@ +CELL,CHR +Tgd,20 \ No newline at end of file diff --git a/example_data/trait_list_test.csv b/example_data/trait_list_test.csv new file mode 100644 index 0000000..a394412 --- /dev/null +++ b/example_data/trait_list_test.csv @@ -0,0 +1,3 @@ +TRAIT +Tgd:ENSG0000010000 +Tgd:ENSG0000010001 \ No newline at end of file diff --git a/main.nf b/main.nf index ff4ebe9..b2b1e23 100644 --- a/main.nf +++ b/main.nf @@ -35,66 +35,78 @@ workflow { // Pass the TileDB array through all ingestion steps ingestion_results = INGEST_DATA(tiledb_storage, list_files, mapping_file, dummy_file) - // Rest of your workflow... + // Collect all ingestion outputs and merge metadata all_metadata_parts = ingestion_results.metadata_parts.collect() all_ingestion_done = ingestion_results.ingestion_done.collect() updated_tiledb = ingestion_results.tiledb_updated.first() merged_metadata = MERGE_METADATA(updated_tiledb, mapping_file, all_metadata_parts, all_ingestion_done) } + if (params.export){ if (params.snp) { - Channel - .fromPath(params.snp, checkIfExists: true) - .splitCsv(header: true) - .map { row -> - tuple(row.CHR, row) - } - .groupTuple() - .set { tiledb_metadata_batches } - snp_results = EXPORT_SNP(tiledb_metadata_batches) - snp_results.snp_tdb_positions - .transpose() // Convert tuple(chr, [file1, file2]) to [tuple(chr, file1), tuple(chr, file2)] - .collectFile(keepHeader:true) { chr, file -> - ["${params.out}_chr_${chr}_concatenated.csv", file.text] - } - .set { concatenated_files } - - // Optionally publish the concatenated files - concatenated_files.subscribe { file -> - file.copyTo("${params.outdir}/snp_table_concatenated/${file.name}") - } - } + Channel + .fromPath(params.snp, checkIfExists: true) + .splitCsv(header: true) + .map { row -> + tuple(row.CHR, row) + } + .groupTuple() + .set { tiledb_metadata_batches } + snp_results = EXPORT_SNP(tiledb_metadata_batches) + snp_results.snp_tdb_positions + .transpose() + .collectFile(keepHeader:true) { chr, file -> + ["${params.out}_chr_${chr}_concatenated.csv", file.text] + } + .set { concatenated_files } + + concatenated_files.subscribe { file -> + file.copyTo("${params.outdir}/snp_table_concatenated/${file.name}") + } } - if (params.locusbreaker){ - Channel.fromPath(params.table_lb, checkIfExists:true) - .splitText(by: params.tiledb_batch_size, keepHeader: true, file: true) - .map { batch_file -> - def batch_index = (batch_file.name =~ /\.(\d+)\.csv$/)[0][1] - tuple(batch_index, batch_file) - }.set { tiledb_metadata_batches } - - EXPORT_LOCUSBREAKER(tiledb_metadata_batches) - } - if (params.export_traits){ - Channel.fromPath(params.list_traits, checkIfExists:true) + + if (params.regions) { + Channel.fromPath(params.regions, checkIfExists: true) .splitText(by: params.tiledb_batch_size, keepHeader: true, file: true) - .map { batch_file -> - def batch_index = (batch_file.name =~ /\.(\d+)\.csv$/)[0][1] - tuple(batch_index, batch_file) + .map { batch_file -> + def batch_index = (batch_file.name =~ /\.(\d+)\.csv$/)[0][1] + tuple(batch_index, batch_file) }.set { tiledb_metadata_batches } - TRAITS(tiledb_metadata_batches) - + EXPORT_REGIONS(tiledb_metadata_batches) + } } - if (params.recompute_meta){ - Channel.fromPath(params.list_traits, checkIfExists:true) - .splitText(by: params.tiledb_batch_size, keepHeader: true, file: true) - .map { batch_file -> - def batch_index = (batch_file.name =~ /\.(\d+)\.csv$/)[0][1] - tuple(batch_index, batch_file) - }.set { tiledb_metadata_batches } - RECOMPUTE_META(tiledb_metadata_batches) - + if (params.locusbreaker){ + Channel.fromPath(params.table_lb, checkIfExists:true) + .splitText(by: params.tiledb_batch_size, keepHeader: true, file: true) + .map { batch_file -> + def batch_index = (batch_file.name =~ /\.(\d+)\.csv$/)[0][1] + tuple(batch_index, batch_file) + }.set { tiledb_metadata_batches } + + EXPORT_LOCUSBREAKER(tiledb_metadata_batches) + } + + if (params.export_traits){ + Channel.fromPath(params.list_traits, checkIfExists:true) + .splitText(by: params.tiledb_batch_size, keepHeader: true, file: true) + .map { batch_file -> + def batch_index = (batch_file.name =~ /\.(\d+)\.csv$/)[0][1] + tuple(batch_index, batch_file) + }.set { tiledb_metadata_batches } + + TRAITS(tiledb_metadata_batches) + } + + if (params.recompute_meta){ + Channel.fromPath(params.list_traits, checkIfExists:true) + .splitText(by: params.tiledb_batch_size, keepHeader: true, file: true) + .map { batch_file -> + def batch_index = (batch_file.name =~ /\.(\d+)\.csv$/)[0][1] + tuple(batch_index, batch_file) + }.set { tiledb_metadata_batches } + + RECOMPUTE_META(tiledb_metadata_batches) } } diff --git a/modules/create_tiledb/main.nf b/modules/create_tiledb/main.nf index f4953dd..a9814f3 100644 --- a/modules/create_tiledb/main.nf +++ b/modules/create_tiledb/main.nf @@ -7,6 +7,7 @@ process CREATE_TILEDB { input: path(mapping_file) val dummy + // Define output output: path "TileDB_${params.tiledb_name}", emit: tiledb_storage @@ -22,4 +23,10 @@ process CREATE_TILEDB { touch dummy_file """ -} \ No newline at end of file + + stub: + """ + mkdir -p TileDB_${params.tiledb_name} + touch dummy_file + """ +} diff --git a/modules/ingestion/main.nf b/modules/ingestion/main.nf index 7100ba9..274410f 100644 --- a/modules/ingestion/main.nf +++ b/modules/ingestion/main.nf @@ -1,25 +1,24 @@ #!/usr/bin/env nextflow process INGEST_DATA { - label "process_high" - //publishDir "${params.outdir}/TileDB/", mode: 'link' + label "process_high" + //publishDir "${params.outdir}/TileDB/", mode: 'link' + // Define input + input: + path(tiledb) + each path(list_files) + path(mapping_file) + path(dummy_file) -// Define input - input: - path(tiledb) - each path(list_files) - path(mapping_file) - path(dummy_file) - -// Define output - output: + // Define output + output: path("TileDB_${params.tiledb_name}_metadata_parts_${list_files.name}"), emit: metadata_parts path("ingestion_complete_${list_files.name}"), emit: ingestion_done path("${tiledb}"), emit: tiledb_updated -// Define the shell script to execute - script: + // Define the shell script to execute + script: def qc = params.qc ? "--qc" : "" def pvar_file = params.pvar_file ? "--pvar-file ${params.pvar_file}" : "" def permuted = params.permuted ? "--permuted" : "" @@ -31,6 +30,7 @@ process INGEST_DATA { --type-sumstat ${params.type_sumstat} \ --maf ${params.maf} \ --mac ${params.mac} ${qc} ${pvar_file} ${permuted} + # Rename the metadata parts directory to include the list_files name for uniqueness if [ -d "TileDB_${params.tiledb_name}_metadata_parts" ]; then mv "TileDB_${params.tiledb_name}_metadata_parts" "TileDB_${params.tiledb_name}_metadata_parts_${list_files.name}" @@ -38,10 +38,14 @@ process INGEST_DATA { # Create empty directory if none was created mkdir -p "TileDB_${params.tiledb_name}_metadata_parts_${list_files.name}" fi - # Create a dummy file to signal completion touch ingestion_complete_${list_files.name} - + """ + + stub: + """ + mkdir -p TileDB_${params.tiledb_name}_metadata_parts_${list_files.name} + touch ingestion_complete_${list_files.name} """ } diff --git a/modules/locusbreaker/main.nf b/modules/locusbreaker/main.nf index 537d1c7..c3d7026 100644 --- a/modules/locusbreaker/main.nf +++ b/modules/locusbreaker/main.nf @@ -1,21 +1,20 @@ #!/usr/bin/env nextflow process EXPORT_LOCUSBREAKER { - label "process_high" - publishDir "${params.outdir}/gwas_and_loci_tables", mode: params.publish_dir_mode + label "process_high" + publishDir "${params.outdir}/gwas_and_loci_tables", mode: params.publish_dir_mode + // Define input + input: + tuple val(batch_index), path(traits_list_table) -// Define input - input: - tuple val(batch_index), path(traits_list_table) + // Define output + output: + path("*_interval.csv"), emit: locus_breaker_tdb_intervals, optional: true + path("*_segment.csv"), emit: locus_breaker_tdb_segments, optional: true -// Define output - output: - path("*_interval.csv"), emit:locus_breaker_tdb_intervals, optional: true - path("*_segment.csv"), emit:locus_breaker_tdb_segments, optional: true - -// Define the shell script to execute - script: + // Define the shell script to execute + script: """ tdbsumstat export \ --table-lb ${traits_list_table} \ @@ -28,4 +27,10 @@ process EXPORT_LOCUSBREAKER { --locusbreaker \ --batch-name ${batch_index} """ + + stub: + """ + touch stub_${batch_index}_interval.csv + touch stub_${batch_index}_segment.csv + """ } diff --git a/modules/merge_metadata/main.nf b/modules/merge_metadata/main.nf index 8c813b8..8e5a216 100644 --- a/modules/merge_metadata/main.nf +++ b/modules/merge_metadata/main.nf @@ -1,56 +1,63 @@ +#!/usr/bin/env nextflow + process MERGE_METADATA { - label "process_high" - publishDir "${params.outdir}/TileDB/", mode: 'copy', pattern: "TileDB_${params.tiledb_name}" - publishDir "${params.outdir}/TileDB/", mode: 'copy', pattern: "TileDB_${params.tiledb_name}_metadata.csv" - - - input: - path(tiledb) - path(mapping_file) - path(metadata_parts) - path(ingestion_signals) - - output: - path("${tiledb}"), emit: tiledb_final - path("TileDB_${params.tiledb_name}_metadata.csv"), emit: metadata - path("tiledb_with_metadata"), emit: completion_signal - - script: - """ - # Create the main metadata_parts directory - mkdir -p TileDB_${params.tiledb_name}_metadata_parts - - # Copy all JSON files from all input directories - for parts_dir in ${metadata_parts}; do - if [ -d "\$parts_dir" ]; then - - # Extract the trailing number before .csv from the input folder - base=\$(basename "\$parts_dir") - base="\${base%.csv}" - suffix="\${base##*.}" - - for json in "\$parts_dir"/*.json; do - [ -e "\$json" ] || continue - - # Add that trailing number to .json file name - base=\$(basename "\$json") - name="\${base%.json}" - - cp -v "\$json" "TileDB_${params.tiledb_name}_metadata_parts/\${name}_\${suffix}.json" - done - fi - done - - echo "Total JSON files collected:" - ls TileDB_${params.tiledb_name}_metadata_parts/*.json 2>/dev/null | wc -l || echo "0" - - tdbsumstat ingest \ - --uri-path ${tiledb} \ - --mapping-file ${mapping_file} \ - --type-sumstat ${params.type_sumstat} \ - --only-meta - - # Signal that metadata merge is complete - touch tiledb_with_metadata - """ + label "process_high" + publishDir "${params.outdir}/TileDB/", mode: 'copy', pattern: "TileDB_${params.tiledb_name}" + publishDir "${params.outdir}/TileDB/", mode: 'copy', pattern: "TileDB_${params.tiledb_name}_metadata.csv" + + input: + path(tiledb) + path(mapping_file) + path(metadata_parts) + path(ingestion_signals) + + output: + path("${tiledb}"), emit: tiledb_final + path("TileDB_${params.tiledb_name}_metadata.csv"), emit: metadata + path("tiledb_with_metadata"), emit: completion_signal + + script: + """ + # Create the main metadata_parts directory + mkdir -p TileDB_${params.tiledb_name}_metadata_parts + + # Copy all JSON files from all input directories + for parts_dir in ${metadata_parts}; do + if [ -d "\$parts_dir" ]; then + + # Extract the trailing number before .csv from the input folder + base=\$(basename "\$parts_dir") + base="\${base%.csv}" + suffix="\${base##*.}" + + for json in "\$parts_dir"/*.json; do + [ -e "\$json" ] || continue + + # Add that trailing number to .json file name + base=\$(basename "\$json") + name="\${base%.json}" + + cp -v "\$json" "TileDB_${params.tiledb_name}_metadata_parts/\${name}_\${suffix}.json" + done + fi + done + + echo "Total JSON files collected:" + ls TileDB_${params.tiledb_name}_metadata_parts/*.json 2>/dev/null | wc -l || echo "0" + + tdbsumstat ingest \ + --uri-path ${tiledb} \ + --mapping-file ${mapping_file} \ + --type-sumstat ${params.type_sumstat} \ + --only-meta + + # Signal that metadata merge is complete + touch tiledb_with_metadata + """ + + stub: + """ + touch TileDB_${params.tiledb_name}_metadata.csv + touch tiledb_with_metadata + """ } diff --git a/modules/recompute_meta/main.nf b/modules/recompute_meta/main.nf index a600bde..b25bf76 100644 --- a/modules/recompute_meta/main.nf +++ b/modules/recompute_meta/main.nf @@ -1,26 +1,32 @@ -process RECOMPUTE_META{ +#!/usr/bin/env nextflow + +process RECOMPUTE_META { label 'process_high' publishDir "${params.outdir}/gwas_and_loci_tables/", mode: params.publish_dir_mode - - // Define input + // Define input input: - tuple val(batch_index), path(traits_list_table) - + tuple val(batch_index), path(traits_list_table) + // Define output output: - path("*.csv"), emit:ltbd_traits, optional: true - + path("*.csv"), emit: ltbd_traits, optional: true + // Define the shell script to execute script: - """ - tdbsumstat \ - export \ - --recompute-meta \ - --mac ${params.mac} \ - --trait-list ${traits_list_table} \ - --uri-path ${params.uri_path} \ - --batch-name ${batch_index} \ - --type-sumstat ${params.type_sumstat} - """ + """ + tdbsumstat \ + export \ + --recompute-meta \ + --mac ${params.mac} \ + --trait-list ${traits_list_table} \ + --uri-path ${params.uri_path} \ + --batch-name ${batch_index} \ + --type-sumstat ${params.type_sumstat} + """ + + stub: + """ + touch stub_recompute_${batch_index}.csv + """ } diff --git a/modules/regions/main.nf b/modules/regions/main.nf index 0012daa..873d11a 100644 --- a/modules/regions/main.nf +++ b/modules/regions/main.nf @@ -1,29 +1,35 @@ #!/usr/bin/env nextflow process EXPORT_REGIONS { - label "process_high" - //conda '/software/cardinal_analysis/ht/conda_envs/tdbsumstat' + label "process_high" + publishDir "${params.outdir}/results/regions_export/", mode: params.publish_dir_mode - publishDir "${params.outdir}/results/gwas_and_loci_tables/", mode: params.publish_dir_mode + // Define input + input: + tuple val(batch_index), path(regions_table) + // Define output + output: + path("*.csv"), emit: regions_output, optional: true -// Define input - input: - tuple val(batch_index), path(traits_list_table) + // Define the shell script to execute + script: + """ + tdbsumstat export \ + --table-regions ${regions_table} \ + --uri-path ${params.uri_path} \ + --type-sumstat ${params.type_sumstat} \ + --out ${params.out}_${batch_index} \ + --attr ${params.attrs} -// Define output - output: - path("*_interval.csv"), emit:locus_breaker_tdb_intervals, optional: true - tuple path("dummy_index"), path("*_segment.csv"), emit:locus_breaker_tdb_segments, optional: true + # Ensure output has .csv extension + for f in ${params.out}_${batch_index}*; do + [ -f "\$f" ] && [[ "\${f}" != *.csv ]] && mv "\$f" "\${f}.csv" + done + """ -// Define the shell script to execute - script: + stub: """ - tdbsumstat --workers ${params.workers} \ - export \ - --table_regions ${traits_list_table} \ - --uri-path ${params.uri_path} \ - --type-sumstat ${params.tiledb_lb_typesumstat} \ - --batch-name ${batch_index} + touch stub_regions_${batch_index}.csv """ } diff --git a/modules/snp/main.nf b/modules/snp/main.nf index 358d390..093eedc 100644 --- a/modules/snp/main.nf +++ b/modules/snp/main.nf @@ -1,3 +1,5 @@ +#!/usr/bin/env nextflow + process EXPORT_SNP { label "process_high" @@ -15,7 +17,7 @@ process EXPORT_SNP { // Get column names and create header def columnNames = rows[0].keySet().toList().sort() def header = columnNames.join(',') - + """ # Create header echo "${header}" > snp_batch_${chr}.csv @@ -23,7 +25,7 @@ process EXPORT_SNP { ${rows.collect { row -> def line = columnNames.collect { col -> row[col] ?: '' }.join(',') "echo '${line}' >> snp_batch_${chr}.csv" - }.join('\n ')} + }.join('\n ')} # Run the tdbsumstat command tdbsumstat export \ --snp snp_batch_${chr}.csv \ @@ -32,4 +34,9 @@ process EXPORT_SNP { --out ${params.out} \ --type-sumstat ${params.type_sumstat} """ + + stub: + """ + touch ${params.out}_chr${chr}_stub.csv + """ } diff --git a/modules/traits/main.nf b/modules/traits/main.nf index 69b2311..aff5f22 100644 --- a/modules/traits/main.nf +++ b/modules/traits/main.nf @@ -1,24 +1,30 @@ -process TRAITS{ +#!/usr/bin/env nextflow + +process TRAITS { label 'process_high' publishDir "${params.outdir}/gwas_and_loci_tables/", mode: params.publish_dir_mode - - // Define input + // Define input input: - tuple val(batch_index), path(traits_list_table) - + tuple val(batch_index), path(traits_list_table) + // Define output output: - path("*_${batch_index}.csv"), emit:ltbd_traits, optional: true - + path("*_${batch_index}.csv"), emit: ltbd_traits, optional: true + // Define the shell script to execute script: - """ - tdbsumstat \ - export \ - --trait-list ${traits_list_table} \ - --uri-path ${params.uri_path} \ - --batch-name ${batch_index} \ - --type-sumstat ${params.type_sumstat} - """ + """ + tdbsumstat \ + export \ + --trait-list ${traits_list_table} \ + --uri-path ${params.uri_path} \ + --batch-name ${batch_index} \ + --type-sumstat ${params.type_sumstat} + """ + + stub: + """ + touch stub_traits_${batch_index}.csv + """ } From 44446a03aebf4585a1449305274b980544ad7d08 Mon Sep 17 00:00:00 2001 From: Bruno Ariano Date: Sat, 4 Apr 2026 19:27:43 +0200 Subject: [PATCH 5/6] fix examples --- conf/local.config | 6 + docs/README.md | 46 +- example_data/dummy_out_ENSG0000010000.tsv.gz | Bin 99047 -> 100745 bytes example_data/dummy_out_ENSG0000010001.tsv.gz | Bin 196948 -> 101214 bytes example_data/recompute_meta_test_table.csv | 5 +- ...66_587dd34ef433cefc4a3d22fbc86b46b9_22.wrt | 0 ...48_33f1383fdf71136063e2ba749ba148c7_22.wrt | 0 .../__fragment_metadata.tdb | Bin 0 -> 11413 bytes .../a0.tdb | Bin 0 -> 2041 bytes .../a0_var.tdb | Bin 0 -> 6035 bytes .../a1.tdb | Bin 0 -> 1991 bytes .../a1_var.tdb | Bin 0 -> 57 bytes .../a2.tdb | Bin 0 -> 6155 bytes .../a3.tdb | Bin 0 -> 6326 bytes .../a4.tdb | Bin 0 -> 6154 bytes .../a5.tdb | Bin 0 -> 12928 bytes .../a6.tdb | Bin 0 -> 57 bytes .../d0.tdb | Bin 0 -> 55 bytes .../d1.tdb | Bin 0 -> 2025 bytes .../d1_var.tdb | Bin 0 -> 56 bytes .../d2.tdb | Bin 0 -> 2037 bytes .../d2_var.tdb | Bin 0 -> 64 bytes .../d3.tdb | Bin 0 -> 4028 bytes .../__fragment_metadata.tdb | Bin 0 -> 11427 bytes .../a0.tdb | Bin 0 -> 2061 bytes .../a0_var.tdb | Bin 0 -> 6093 bytes .../a1.tdb | Bin 0 -> 1997 bytes .../a1_var.tdb | Bin 0 -> 57 bytes .../a2.tdb | Bin 0 -> 6163 bytes .../a3.tdb | Bin 0 -> 6350 bytes .../a4.tdb | Bin 0 -> 6170 bytes .../a5.tdb | Bin 0 -> 12937 bytes .../a6.tdb | Bin 0 -> 57 bytes .../d0.tdb | Bin 0 -> 55 bytes .../d1.tdb | Bin 0 -> 2031 bytes .../d1_var.tdb | Bin 0 -> 56 bytes .../d2.tdb | Bin 0 -> 2044 bytes .../d2_var.tdb | Bin 0 -> 64 bytes .../d3.tdb | Bin 0 -> 4509 bytes ...322102641_4acfe9cc6c6bf6a9b9d0395481c3ce54 | Bin 0 -> 246 bytes ...322099133_20a54484a947344b0f0c80c9d194a2da | Bin 0 -> 271 bytes results_ci/TileDB/TileDB_test_ci_metadata.csv | 3 + results_ci/TileDB/dummy_file | 0 .../execution_report_2026-04-04_19-01-35.html | 1046 ++++++++++++++++ .../execution_report_2026-04-04_19-07-01.html | 1079 +++++++++++++++++ .../execution_report_2026-04-04_19-07-26.html | 1046 ++++++++++++++++ ...xecution_timeline_2026-04-04_19-01-35.html | 225 ++++ ...xecution_timeline_2026-04-04_19-07-01.html | 223 ++++ ...xecution_timeline_2026-04-04_19-07-26.html | 223 ++++ .../execution_trace_2026-04-04_19-01-35.txt | 4 + .../execution_trace_2026-04-04_19-07-01.txt | 1 + .../execution_trace_2026-04-04_19-07-26.txt | 2 + .../pipeline_dag_2026-04-04_19-01-35.html | 77 ++ .../pipeline_dag_2026-04-04_19-07-01.html | 53 + .../pipeline_dag_2026-04-04_19-07-26.html | 53 + results_ci/query_region_list.csv | 3 + results_ci/query_snp_list.csv | 8 + .../snp_ci_chr_20_concatenated.csv | 8 + tdbsumstat/utils/create_dummy_data_sc.py | 263 ++-- 59 files changed, 4255 insertions(+), 119 deletions(-) create mode 100644 conf/local.config create mode 100644 results_ci/TileDB/TileDB_test_ci/__commits/__1775322100666_1775322100666_587dd34ef433cefc4a3d22fbc86b46b9_22.wrt create mode 100644 results_ci/TileDB/TileDB_test_ci/__commits/__1775322100948_1775322100948_33f1383fdf71136063e2ba749ba148c7_22.wrt create mode 100644 results_ci/TileDB/TileDB_test_ci/__fragments/__1775322100666_1775322100666_587dd34ef433cefc4a3d22fbc86b46b9_22/__fragment_metadata.tdb create mode 100644 results_ci/TileDB/TileDB_test_ci/__fragments/__1775322100666_1775322100666_587dd34ef433cefc4a3d22fbc86b46b9_22/a0.tdb create mode 100644 results_ci/TileDB/TileDB_test_ci/__fragments/__1775322100666_1775322100666_587dd34ef433cefc4a3d22fbc86b46b9_22/a0_var.tdb create mode 100644 results_ci/TileDB/TileDB_test_ci/__fragments/__1775322100666_1775322100666_587dd34ef433cefc4a3d22fbc86b46b9_22/a1.tdb create mode 100644 results_ci/TileDB/TileDB_test_ci/__fragments/__1775322100666_1775322100666_587dd34ef433cefc4a3d22fbc86b46b9_22/a1_var.tdb create mode 100644 results_ci/TileDB/TileDB_test_ci/__fragments/__1775322100666_1775322100666_587dd34ef433cefc4a3d22fbc86b46b9_22/a2.tdb create mode 100644 results_ci/TileDB/TileDB_test_ci/__fragments/__1775322100666_1775322100666_587dd34ef433cefc4a3d22fbc86b46b9_22/a3.tdb create mode 100644 results_ci/TileDB/TileDB_test_ci/__fragments/__1775322100666_1775322100666_587dd34ef433cefc4a3d22fbc86b46b9_22/a4.tdb create mode 100644 results_ci/TileDB/TileDB_test_ci/__fragments/__1775322100666_1775322100666_587dd34ef433cefc4a3d22fbc86b46b9_22/a5.tdb create mode 100644 results_ci/TileDB/TileDB_test_ci/__fragments/__1775322100666_1775322100666_587dd34ef433cefc4a3d22fbc86b46b9_22/a6.tdb create mode 100644 results_ci/TileDB/TileDB_test_ci/__fragments/__1775322100666_1775322100666_587dd34ef433cefc4a3d22fbc86b46b9_22/d0.tdb create mode 100644 results_ci/TileDB/TileDB_test_ci/__fragments/__1775322100666_1775322100666_587dd34ef433cefc4a3d22fbc86b46b9_22/d1.tdb create mode 100644 results_ci/TileDB/TileDB_test_ci/__fragments/__1775322100666_1775322100666_587dd34ef433cefc4a3d22fbc86b46b9_22/d1_var.tdb create mode 100644 results_ci/TileDB/TileDB_test_ci/__fragments/__1775322100666_1775322100666_587dd34ef433cefc4a3d22fbc86b46b9_22/d2.tdb create mode 100644 results_ci/TileDB/TileDB_test_ci/__fragments/__1775322100666_1775322100666_587dd34ef433cefc4a3d22fbc86b46b9_22/d2_var.tdb create mode 100644 results_ci/TileDB/TileDB_test_ci/__fragments/__1775322100666_1775322100666_587dd34ef433cefc4a3d22fbc86b46b9_22/d3.tdb create mode 100644 results_ci/TileDB/TileDB_test_ci/__fragments/__1775322100948_1775322100948_33f1383fdf71136063e2ba749ba148c7_22/__fragment_metadata.tdb create mode 100644 results_ci/TileDB/TileDB_test_ci/__fragments/__1775322100948_1775322100948_33f1383fdf71136063e2ba749ba148c7_22/a0.tdb create mode 100644 results_ci/TileDB/TileDB_test_ci/__fragments/__1775322100948_1775322100948_33f1383fdf71136063e2ba749ba148c7_22/a0_var.tdb create mode 100644 results_ci/TileDB/TileDB_test_ci/__fragments/__1775322100948_1775322100948_33f1383fdf71136063e2ba749ba148c7_22/a1.tdb create mode 100644 results_ci/TileDB/TileDB_test_ci/__fragments/__1775322100948_1775322100948_33f1383fdf71136063e2ba749ba148c7_22/a1_var.tdb create mode 100644 results_ci/TileDB/TileDB_test_ci/__fragments/__1775322100948_1775322100948_33f1383fdf71136063e2ba749ba148c7_22/a2.tdb create mode 100644 results_ci/TileDB/TileDB_test_ci/__fragments/__1775322100948_1775322100948_33f1383fdf71136063e2ba749ba148c7_22/a3.tdb create mode 100644 results_ci/TileDB/TileDB_test_ci/__fragments/__1775322100948_1775322100948_33f1383fdf71136063e2ba749ba148c7_22/a4.tdb create mode 100644 results_ci/TileDB/TileDB_test_ci/__fragments/__1775322100948_1775322100948_33f1383fdf71136063e2ba749ba148c7_22/a5.tdb create mode 100644 results_ci/TileDB/TileDB_test_ci/__fragments/__1775322100948_1775322100948_33f1383fdf71136063e2ba749ba148c7_22/a6.tdb create mode 100644 results_ci/TileDB/TileDB_test_ci/__fragments/__1775322100948_1775322100948_33f1383fdf71136063e2ba749ba148c7_22/d0.tdb create mode 100644 results_ci/TileDB/TileDB_test_ci/__fragments/__1775322100948_1775322100948_33f1383fdf71136063e2ba749ba148c7_22/d1.tdb create mode 100644 results_ci/TileDB/TileDB_test_ci/__fragments/__1775322100948_1775322100948_33f1383fdf71136063e2ba749ba148c7_22/d1_var.tdb create mode 100644 results_ci/TileDB/TileDB_test_ci/__fragments/__1775322100948_1775322100948_33f1383fdf71136063e2ba749ba148c7_22/d2.tdb create mode 100644 results_ci/TileDB/TileDB_test_ci/__fragments/__1775322100948_1775322100948_33f1383fdf71136063e2ba749ba148c7_22/d2_var.tdb create mode 100644 results_ci/TileDB/TileDB_test_ci/__fragments/__1775322100948_1775322100948_33f1383fdf71136063e2ba749ba148c7_22/d3.tdb create mode 100644 results_ci/TileDB/TileDB_test_ci/__meta/__1775322102641_1775322102641_4acfe9cc6c6bf6a9b9d0395481c3ce54 create mode 100644 results_ci/TileDB/TileDB_test_ci/__schema/__1775322099133_1775322099133_20a54484a947344b0f0c80c9d194a2da create mode 100644 results_ci/TileDB/TileDB_test_ci_metadata.csv create mode 100644 results_ci/TileDB/dummy_file create mode 100644 results_ci/pipeline_info/execution_report_2026-04-04_19-01-35.html create mode 100644 results_ci/pipeline_info/execution_report_2026-04-04_19-07-01.html create mode 100644 results_ci/pipeline_info/execution_report_2026-04-04_19-07-26.html create mode 100644 results_ci/pipeline_info/execution_timeline_2026-04-04_19-01-35.html create mode 100644 results_ci/pipeline_info/execution_timeline_2026-04-04_19-07-01.html create mode 100644 results_ci/pipeline_info/execution_timeline_2026-04-04_19-07-26.html create mode 100644 results_ci/pipeline_info/execution_trace_2026-04-04_19-01-35.txt create mode 100644 results_ci/pipeline_info/execution_trace_2026-04-04_19-07-01.txt create mode 100644 results_ci/pipeline_info/execution_trace_2026-04-04_19-07-26.txt create mode 100644 results_ci/pipeline_info/pipeline_dag_2026-04-04_19-01-35.html create mode 100644 results_ci/pipeline_info/pipeline_dag_2026-04-04_19-07-01.html create mode 100644 results_ci/pipeline_info/pipeline_dag_2026-04-04_19-07-26.html create mode 100644 results_ci/query_region_list.csv create mode 100644 results_ci/query_snp_list.csv create mode 100644 results_ci/snp_table_concatenated/snp_ci_chr_20_concatenated.csv diff --git a/conf/local.config b/conf/local.config new file mode 100644 index 0000000..dfa2bf7 --- /dev/null +++ b/conf/local.config @@ -0,0 +1,6 @@ +process { + withLabel: 'process_high' { + memory = 12.GB // or 8.GB; stay under your ~16 GB machine RAM + cpus = 4 + } +} diff --git a/docs/README.md b/docs/README.md index 89db0d4..2d1e0b6 100755 --- a/docs/README.md +++ b/docs/README.md @@ -4,23 +4,29 @@ TileDB-sumstat is a Nextflow + Python toolkit for scalable ingestion and export ## Table of Contents -- [Requirements](#requirements) -- [Pipeline Overview](#pipeline-overview) -- [Usage with Nextflow](#usage-with-nextflow) - - [Ingestion](#ingestion) - - [Export](#export) - - [SNP-based Export](#snp-based-export) - - [Region-based Export](#region-based-export) - - [Traits Export](#traits-export) - - [Locusbreaker](#locusbreaker) - - [Metadata Export](#metadata-export) - - [Recompute Metadata](#recompute-metadata) -- [Nextflow Modules](#nextflow-modules) -- [Testing the Nextflow Pipeline](#testing-the-nextflow-pipeline) - - [Stub Tests (CI / no data required)](#stub-tests-ci--no-data-required) - - [Integration Test](#integration-test) -- [Test Data](#test-data) -- [Support and Contribution](#support-and-contribution) +- [TileDB-sumstat](#tiledb-sumstat) + - [Table of Contents](#table-of-contents) + - [Requirements](#requirements) + - [Pipeline Overview](#pipeline-overview) + - [Usage with Nextflow](#usage-with-nextflow) + - [Ingestion](#ingestion) + - [Required Files](#required-files) + - [Parameters](#parameters) + - [Example](#example) + - [Export](#export) + - [Common Parameters](#common-parameters) + - [SNP-based Export](#snp-based-export) + - [Region-based Export](#region-based-export) + - [Traits Export](#traits-export) + - [Locusbreaker](#locusbreaker) + - [Metadata Export](#metadata-export) + - [Recompute Metadata](#recompute-metadata) + - [Nextflow Modules](#nextflow-modules) + - [Testing the Nextflow Pipeline](#testing-the-nextflow-pipeline) + - [Stub Tests (CI / no data required)](#stub-tests-ci--no-data-required) + - [Integration Test](#integration-test) + - [Test Data](#test-data) + - [Support and Contribution](#support-and-contribution) --- @@ -325,13 +331,13 @@ To run a full integration test that actually executes the ingestion and export c echo "FILE,CELL,GENE,PHENO_VAR,N" echo "$(pwd)/example_data/dummy_out_ENSG0000010000.tsv.gz,Tgd,ENSG0000010000,1.5,4000" echo "$(pwd)/example_data/dummy_out_ENSG0000010001.tsv.gz,Tgd,ENSG0000010001,1.5,4000" -} > /tmp/example_data_table_ci.csv +} > example_data/example_data_table_ci.csv # Run ingestion (no container – uses local tdbsumstat) nextflow run main.nf \ --ingestion true \ - --file_path_ingestion /tmp/example_data_table_ci.csv \ - --mapping_file "$(pwd)/example_data/mapping_file_test.csv" \ + --file_path_ingestion example_data/example_data_table_ci.csv \ + --mapping_file /example_data/mapping_file_test.csv \ --type_sumstat qtl \ --tiledb_name test_ci \ --ingestion_chunk_files 2 \ diff --git a/example_data/dummy_out_ENSG0000010000.tsv.gz b/example_data/dummy_out_ENSG0000010000.tsv.gz index 79e5937304acdb66bd04259fddec123ba229943a..0d94e0a7395cf7a602b3fd2583cf3d7dd1287612 100644 GIT binary patch literal 100745 zcmV($K;yq3iwFpyL(yph|73M-ZFygBb#z}vPE$uPFfcGMF)%PNFfMd+b^v_6$*!e2 zmYq4adKXKuZ(`;MNHG%xrN2OyDo`2-QbN+C-oDniHjf+6fEH3_2F=ayJ?H<)&E2f# z@Bi(8{rJZ}|HnUn{MUc}_y7Jc|L6bv_kaHQ_y767fBake@o)LZzy7EH{NrE${Num; z`QtzQ@gINu`;Y(opa19Ie*Eu${$C~i_z(Z-U;mN#Pv(F9`1Ak$e}3du`j0<;{Qbv& z{cr!Pq`$HJ+aG`X`;VWc)spLv^e?68zoc1LZflh?yZ&s}(Pl2W<pMYbCW*r+$6fdyh2uvp(`z`s|Y*HCugr27T16&+zB@3@r5-{?ud8 zSIYbx{n6k#g0;>YpvzFCU zXFKpmd2MYHy|0|L7uMsq>%|VefX}aI`L{p)1bW^*@qo0n7Y35NLA3=|oJ%g#Y-cxOxPmuLywdHo5Bk!{bO3OMLS*LR~9tWF6Z$n3Pm8o~L`+98xe!hPH z=OM7vYxw(*y0iiLIaAL&*FJR)^lb_sA@#obr?cxLt?Iv~129t83)k+~8H@wd^%D3? zenrot&}Y4mf|hX+KnvFZY0F88TQtx(~FpTwRvzHvffh7vo)PKJsN#f zeFmMqt#?*aDTbyj%{SutVkraOecos#!R*HPa3NQ0YdzE5SoLlaAfCcRR{UOq$V z3e0&cK8%)qMJ{avZH}g^N1MfWNXnh}HR{%yLVT#Hn)g&R?nTeN`HvJl^__Hxn_l5-D=6UX zJ3ea}@wsS|mlZ!p@4M@L>QZQlr?CCVThlMo_0=*jVzCiwA_gg5I(ca9pVkOXiqX&a zkaf7OTbkv$rM0!a&w9}p)wD!wv1kf1D4%QlaS|KlFubj?K7v3Vz8g?nos9`~Zc;+&Z@H*IQevV5sXH-8 zMx$M@6k2tZ^?H;@z0QMzOt}sO%ShRpB7sA_?%=8%CG(94GESFLR&cCbO$r&j2fc4y zYo%t+sLzc)jZ(I>*%=C={S}#X8UES`TE$_|d-k22^e7Z^l<(+J=-c_MEBt7aI4&ig z5r0otMOJpn>D2{V`~Zs*6ds|H4q`@or7DW2*G^#Roj?}i@2kC}eY9I)^XYQpkJ41k z&~IzH0BhuHi#@dXC%TiXBO}{I7n`!Wq3dM5J^^3vii|8B8MP4=W&Lw>Ep&2bJ#f8L z?+!wbZl6LNim;^bI`ytQFH4_z`jXd!Qef8qQmoAV9h`sapl8skXT0W7V^T0C72UUQMgZk}MgN_YObDc#gM^}PO_O7alUq`W^ zEJi1C=rj3tP=cY2p=Y8Pp~U}nL+Vz!k~Wr+v?(`M$r5#t4!=%cCxP{?nTeCSko41) zsg0;CbtmYmp4$1kwtSAf(C5^i)q(2I6B((>Uneqhm4EzDtgp2=GtwznIF4N@netpb{z-k|>=FJ8Q`J%3CZ`u)xPgtDxnO~tO#bzNU4LYr+}?}DY@ zg}zVsu2Pv!F{_J@;?0-OR#eg_;2Uxa6^AeKcIBxQ``K}w0z2uk(F#fWZ~NI)Xz8n| z(9&rtY!14w<=T}5C{igPa#x%)k@px}ih9dYYIOfUMIxm()5VlxnPPV3idE&QzxKQ% z^dYXcf|gN2s1_ZrU={HS!II6cpYG^HCZxc-D|xjmjisp6r~Z3ZIM5a1C_0oi4wpa5 z$zPj+?{;03EYC$bM)jLdq>GExSTRS(u0>@($kMbalr4_VKk z%yfV0XDC3tK7cO4>ou@EuR+f<(hh|P0+Eiq_Cm?t9?Br}ly&_nS6@-Z4<0sMpv@(p z{c8B;ircEdbkbh$fvn(q2rT^+DwiS~K^so_2|0_>gY1o0N#j*Qk+-Uvs&2ArrP6!p z=GJ$2Y2|LDHjDO5+v(Zn)2mEZuD~*K1tm{PH~1=~AvAAv*5ayGVnpp!Nn4c0mq?wJ zIOq#^GHqYgIxdyVjg)%n_Iz%Gau>5KBX?KaozX38nVON~pi6J^M|v2_i0Y!FqOk4c zSqIn=vnhR69^srq+nG*>HiC}kv%R9SQBzJ` z5idcBGXt>*bqw9&A z+EGSVz*N?tXs}4ABYCQ1hl+khXU0{J;u-N(|GNHAx9RJ4q!##f3|Pi7pmg=m;ci>n ziCL+dUx0SJ5;x@@gH0AGf&;-`$|V$$eXlDZltF$ur`u1jQ=oL~PSv!GR83DXB?oqX zMqW4P7omIJ zkyUAtvH)!}{XpG5Cl8zS?hTI(S8Aby4f;D8wEjlx?1cV&n5}B-&kvk)=@_ zWF5A4+EdvH*eq^m`?e%u)W^~tF!Y>hA`ufah5uI4%U$4D)<_eiU8#ygV%CqQBB5PVVuMt&2%0>3=#%WoGdKO9?8it;@L5T>dH0pH zjQWYD*i63Rno=*FzdX;0RFKhk)p^iyD|$uV2?`H-+KQ3gH-Up9QA@#Fe@T~leg#F2 z@%<6BjE06PI7d5Oz@yQaX!^Q}^#0kpyOlvIZ|U4belVm83gFtV3d7AeHdhPNh|Q&` zfB6{{d%0c%%Se;TisqW`;H*NLVN-Xr(^1IHDn0x}ne8$`!M}R)*Xb@5|CzRq_MH=p7W-8=0_Zu&@E6eaZL2LmQ0LN1~7XmkR#6&0DL?hGe0 zq$`6KlyXM>>#xtC2YAh3Sb7HYvo`j&n`uf;v;~Poc}2H^L4aOO=33UJ=pK|lfU*OH zW`adbVe}gEpzK{_tnHaDD#%{3iDkqlZMT~Ia|#Di-&c1tjS@C#Egt&lLRP?vCSM^} z)vZ9kM|;C@Z8AVmVABQ1CI0L)mOD0Sbutxt1)Mbk&xRcIn7z~VYB>dSkVN{2>yYXfFppx+QqNP+ zeq>0c6F|!3&`O(HC!O%PfAbg6uRRHwtH zFrb@&Mklwsi|$-}a1>`JW2I-alh){!vS}IHQ9mc1FHgi^!|1 z!q@fgx}+5dTh$p^9u3icQDh^;Gdq6etqOGac2qGPIEB$7@)48t5EK~bFFWSwG?Fap zq3dDA5M6n^{z~@wP%!jg>vUf>=@reu`K(?X9{(BwC5Oj9Xzp}B^~^Ua)oJRa81n9L z6LIR-X}?7C(cxe!4gqLTLJ_k=-k)NlUKx|y*E5Jl-(L?Q8hs4IRF5PVtxMZN7Z_<} zi;+Y!c41S)a$h)`oXll(j8i$|s!2dv)H*636vSQ$iz=?WucD0bnp4H*8Wqh)0Lbe5dWlnps(oAyqF9+FzOI2W>+LmvG{n<*dH%>idpj89_l; zLW_1`r6Lp>e)SEd~1Y>y0FgfJ;nyKqKCej5&Q6ForFQ&ss~D= z@p(ghO~Y9Fh9KYbiPvS&0b)2DZKp7FGzm=#Mk-yLqLJML_|qOz;!=Hg?ilE5GwZzS z)4%WOdn>Svtw4nF{5>`G^SOcbOzWy7sj-9%1kl{yxh&`K*i3BrovB2j~r zN4%%A3ODzm+|h1zcQ^XDI#TqJ6^mkI-T?J2uB1{uj&eVz$BjkbCtB5xXZq#(+?ZtjWn&22FI4AwqX#R6IaTo`) zsP!C1fkL>4Fj^iMm1-h)c_`I8UCzQBF#GX>mT5}jdETnSOGGdRw!j)Ytudzy+BwRu zne6CDypalf-J4NMZ$=tMPFGa4>Og1RJNgKz#LQZ0$tLD7vFDN)~jcZWx!A(Q9kqFapV%xuQS|=pc3nLiU^PZ#@Vy$v`w{yG;t%tlPS_ z17Fh5P()P*qv%wgO&()Qe>HiGE!A#h3Z1+WXzcNZl`Sx7Hte?na8+_Lm^lQla3<0V z<}bRZTg>g4p$yP1%4oMTtXJk(YrcAZEZy_dHA$JrX;TE!w_%<&;_mF!Xfx3W)_x0Y zWmCQ3E=FZUGy*uIh)9!;tVJ*Oc~xp#-ADl~BQ|mp9f-FIc%6r?P?w|GpjWiOE9mNS zWTteTwbEo|Eay6Z8Cj{FsbHqp^8`?pbRIGqoA{Jx*gU?p+R!Hekv z41`SO&R&ed!CE<40e=Vvle4S@`Z<;bEAcCBWqH!pO2+G5z9@kh^)Q5YB39^flz=NS zm5Jgom9c9d>ai=x7)i4g?3JhiaHWmo z=^V4kL;#>IF?Wna$*lE@NP6pg8mNoq%2Zf7Q)zUST}NgHDBxx?Fo>v=2dG_LeYyi? zY*-KjK>o{S5BU?2P3SY|Q83heUJXjQ@0g^eV-o3EG!_?-8I=I~;x%x4`V-?XGzsW19w+DkS~ia^fO|1X#)L z5hQ1H(fm;jQ9z>SfS`hr7!=j9^M6LY2z)k?3xJaIzUMb`Z_#VjQHyK} z*pgCMre>;eBadfDs0*@iwdqN|-huAjJ6+T=-~kmBwNnJ`eUIRD(oc!3Yn+}{gR4TF z>djJU?D7^Ey~K0MG~0?UM?Dc`zt77l-+)S%ehL}Djpxm6(ZE@}^}x*1+oFTH3D?ns z+u*i9;#cr8GrC=jHJO_hx>D_#sI@N=&}adtWRMb&|HQXfnnHp+51LxvQkyOJLd_8O zf;;^BAEn7TW`ITOPlz%Tc3d zVd}?4<-tdQjg`G>2Y{|&K=>R%<#JDDSsnnLa&b4Ov@xgCJ#N7gy&@G8*=>B%2f zhjcO+sw2fC(5X;9uTM(x|12Hz_@2X!)9WVXbLOL?Oj8lOi974jaa3tAITcEe_1?FE zjMak)rR9vlb`&i%2= zD{a&R56X02^vGshm?RfqQ5~0`V+X3fuQPe@luVhO6uD;7ccEq_6FHf%jtT9>(LR`R zW2}>Q37t<@PO5I;%tbHdeJ`Z10+!|B;OG;i;JR#WFnx&b1Sl95pz0WS)Bf(>j>-Z! z`Xgo`F|P)?0h&cw)_%W*zSiC8Wf>_mjm5yxbFL<^ex1|J(ePcU-LIX`dnjiR4H>OC zfzzjONYidEL* znF;MAJo}II4Ja2BG20t1qvKP)n1Q)O0ju)Eco|4H)Wu0(W74N^gQDpPDOZo?b{t0) z-`X67F7v0#)b*Z?72tl?|9v4i|F+f?HT=&Fc0#>3N3DkyeoDvMp zpC-BhCZ_2%W9XrqL~BqjBrNrP6#5rTM9Ub8m(>pRg7^-7rjiQAXVNc%EDLBtpTX5* z$?yo77J;wZJstuRI>#mL+~?Im#ed@{S33h~`ZWG73M}9yM>3S!={V2`RV@19x*QL9 z1L3g8cbv}!D%(Gtn;JK?rwwqWH&P2JCu(L;|ps z^ctx8qsoCYqM$%dzn*CVw1~U4xAxf!`hsO2_6xwqv;aszA&C-h@*VYwX`2;)iVS)T z7%i}|7Vk{iJD?ARMuK*FYdL@{z54ASOJnFTf~Lw0j%}y=vmeUe=FoRWZ8{|>wgaO^ zKt3wmba0%%Oh{}M+yoD4m-ke)x@gjseS;84{!Dbfc()w(p5snhU2`vLs~ z+F&cenhK_@{!Q9Ro<}4ws!u@SxQD_lKb)`}^B(|Y$`fd|f|hbFY1B5Kfyr!|F)A~9 zjo1X;!q02wsD8-Z)_T4Q{rx*iv-E?|h3BTqNRXeYo*9Ld6CyJ`ExmIc*l1zEAmd|F z!2^VMEC=+xVOHu{&F59gmc3d_EI;_I!yE8W8Nk9~py0SoI;8}jN0x)%-GNUwdWj62 zS^z?;s}3sM{l=A&~HNCT+7kq-`2nIXVLk03j152QiP^NYqJ_ARxKBQWvGmD0H0;WyEk# z%a9cPm;AgVkW9H+Y$+cC5WUQ3?@$DijjCwK(RP~vBx*!K|P_q z;QlP2*q#`X@&yqo2e$=mFuDSXQ7A!r2TqODZMuo0s*{tzE>PxzenRuiHbqjChZT^5 zfU%zwFl9Hl-V)2R59lri3Wp?wI#nMk1$YK_Jc;{eY~7-d#Hh|xUDDhh+6PP@baEPT z;o00u9zOlm=}jI!=`Bz+%%I;i1aH_-)b#G0k$Y?aWs7dZa#UiYP<~|i-2q=i^Z=EX zYQE}kDu}kf=$UPQh>3Yc77jG8bFQJzjum*aM4yETfQ}mx+=-(qZewGVKu!w~bb~&X z&LSMtpVvcW26m-P%)laju5lvCfPBsB*6)E8Wm0SCavv_3BW)WDM>k{$l@2cNHnBuG zRRI_I90=zxXaLJoO3oeJ&jxC%)aa!Is7@O_6ZIJZ(}DN(L_4nrF?V@p!6*}kKd>`u zo7do>*ZV!NJnx~TqhiuHQ=P(9fhyo+sMgThu-1a_i)@}Q{1CFIV8nHg3&@jD!+4`P z*PXqyJlQ*O(H%_KEfb7phlgw=7x-Q64W{3qh7M9@1CvWo4+8AVUuy=I44&Wfa`-|= zu#Bigqus|(KY-s}CjU9aS;o;^e}%SRIR|2d3QULAyv~q>fiSl_AhhyC5h?lKH zREfcOIv`~*n|i7gk1M+Wx*Jry!|iX8sE|y7lOKSwXFXaDY+Bm0fV`hIHT{=6TjNar z8T}naDHpV0mcaw|Vhu9)?=sXH3gzB{X2F;Pq`flR@?ov+Y#PMms(-(SaZdqR9>M`S zPVNn4PTT-8%%gW)qDF_1n&)4?pNiaW#sKMhi-(n zJC7HtHqG~QMf%o6icDuOgoZXUOQd*;)jFJW#a&W&zY5R4yHuNjN1{coI+ifiE4lH6t+oY?R_b)Gds* zGLr@-^lo)zPAd#vO39-Lm>@4Jw_#MDLlF-cdI3?AVT(N(+xj}?3u|9ae+T5cC(|8r z0U#|3x#7{guFMt(PCWG1d5b;;UM5PK@sBeK) z2_!Xiw-wZyVSW?6g7U%pqsU~vmmtnY($;{o0YjzS{EfuRxVa}R1E4^j?k3){n$$FW zJ00W!tqS@o1?pbt9H0Vc-6#49f#s_7e-U!r#NUa?v&q7e7n#|N^5OynD0WzZN3Q;zd9Rw62RsBt)Q9tS{Z5}IXY4qGeM@_$lB)JU>yV6a_31arNul1-hK1!yIDWn>AW_3tEc# zXx*?9M=q2jz}Z(I6>r6zSB zjV_dFy7~#}Nw+{v>+62nF?`T#--qghD2q;c_M~fs3>M7=F3k?0GzKum44?s0q4={; z7^Nt$fZ@;^d!NO0hs5e>pvND`Mg;y$Avw{W(qnL4*5I@h6p$+MC+0mc!d&01K(KVv z0bzfr)U5NZ3R%kCV%73A6Li&i`oVl0JXUA+kprX!khxkrRnNf3$?yPLlU!RJMoJLI z=y^;)U#JE?>$d0}k#9OzvpraFrt`q(4+5b{9& zFFqIp&DR)8xzrVtSvn>I+dCS;`uO_yRirXfa1S1S4P}YW2ypul{_2o^fKtLuavqKH z%Rs=@z}{!SeSXmlv<#&sKn?Le2ceG~ngMH%yHO<_6C$`!BeXyV0+9F%OpU1O@^gk(Z$;Q!<(n$f)@^GIY1-9Rrc*I5dk3lOdL!qBT0X($P3r z3^Lzc-g$3D;0cEBjnl_7Q5z98TJJ$S&{l!TfM}1hi9o<-iv|y71ln^c2gD2gClY%l zZut$NRGxoBD7Ex)oTVT6MRFWOFTsTcxK)=m0XWm&(0v?|cL||0#cBQ|^fOnp>KU_R zX6sJ^Sj*Q*V0q91XhV7av>ENYwXgG+OSDt?#t^b->e5B@UStep(Cc4J0K_BNnw?33 zGN?CPCVhkK(*fC|?-@cpKv9vPL|d)9;%EZvmR8P#4mlSUWqstZ!4LOq(Oir|d~Cl!34ddy2hpwI>W zleA7cay44Dj15WWI3&dq?^WqPt*l6ssznhtZH>s!fjHAt2j5Ds$uq=B@qvEw_zi9g zJ%+o}T#pvuPnx!&pjL09E88FVk${~M6YTAv3aYKb$1lbKDRgM-dpo$fWzbGd@KLkCxsa1_*@Fn{Esbq+%G#3CswiF)d#?>R7b$m`i(j`s^@*>$p)% zO4$C4orS+2GAvRed3_0^)hk8119QzEKEWXbI0}JrBa{ z#t3K`#WbNk1`o}}99Oax9%Cp(1;;ko>%h+Ge_7KMWyqkLCnz%>LqQeR#qk``E9%?| z23j7NyFQLHt)>lV=ZGgPm?l* z;+pZ407Q^I^m$`z6mlf+0LCLsqPX@9aLN+hUNFzdaG>*HF8f=F?vBwc&p43sFjBGt zM6W{Xn$dV4XaegWZ`!cn9^G;-gFJtsC&}M^77-R){qs_SYti*C2eN_; zroJ&e6V6x-s{v$njJAc5F$2yAGXOwi$vZ4V5Hp~wgla7~xTl`QzA=?715>$<>f{7m z5AXrG^k^1nqndceeWec-Em%APXw4c+utR%F*#)38;<--qGc@gcH?llj1+GhnjR8}*lE0!kHa#$RiBrp+%qugjo+J5%b>_2z+g0xL3xmn2eNC2 zG~SkbjXZ@G+y{&(CRD-1r5((iWl~NmC!$bL_6U~n?T=Q#({*M z6Bkp5-EZLQftNBnWkQH3$_n*hGMItJaTqf^xmfF)jkP{}tghW{?JH8$Y*U*Ae<4Jh zNy|y>$A+NqgrPzauc*<%3Oi?_BQg%$djsKmOO@Vnfn~%6{Bw37$(5ZzID{n`NCh$? z#`uc|Tw_e&8Pv4IOF4{M1!cV*)a9Y${024ig=5q5$V5^Su1JsR?<&kRcNpC1LjH)p z2lJtQM3)dYqSU3k(QEW2>HDAr%~7HI@I07-q52Cx6c{QaJh^V2d(hSc>N@E)9L;cr zC3WnqE($;dgLt2>cEL?AvR=T!)DOWQEo!i3tGd z_E5eu*{{B+=qn(Yz%{D-HR?)4fK*p!Xs*xudevK6eAQ4E$L(SSRaGVHFmf~<**`8w z_m>$I&Vx4Q`)79@5GT|iqcBXL{c-=d*`Fb5pewxYIR&r{I= zRod-{s1C%Hx(o3dg&)R9(+qx)RZocPxS@P3F3A*csn6Pc_$R+fe2MP;0+b5 z0A}lqI7kd+xkBp)+*4j*<)t(6Y&4~>0Ig*(6v=hma3s^nmJFb_cj8!SAYlzNVT!uW zB%huJfxC?G!BtwHfl2@&dRGxV$D26bd_~an)Yo@gn+UjSK$cqALoJh)!3fgua{7Q= zHVO(Tt5}R2!YtyU2tnYF^L7MkUuSU~@vU-F1>A<%MCi*a(w;#d=n>%jfE9P(s)740 zD{6*w3nzuFkA4J9?&mG-UGUcjXx;_-067PeT&J0$j?1@&`j}BtC1&wsMfFTAiOjb? zLQpG#FVLw*V}1Wd;a^$TJ&n}zR4+5s{LxP$wp24dLKMKvOd4Mt|7gdBAxo#QpXZsN z?#c8CgaIOM&oCRT8G8Hwv}2c8 zKwyM5VG2APjoN7LQ7A79DqyD)7G(N~kX3EJh2c)YzL6pPjrT!<&B-wGP+t6 zT<%)f|E^jHm87LUM)xOwDm*g2XNJf)V zot={oxety@Tj7GDz@U1J--S9tVC;~3cDH|~bC)nb!fk|VpCa(t$FL96FJ}e&FexIh z68wLHTB;?)L3_x5f&Wi*v^*jVWzsVwpEd;v84+e4`kbgKfJA*A$Mt1U$MTFqQx0zv zbi{bnhzp^P8%|*~rTIv<{|FZfgWL2N7#AhyCZt8&;LJ%lJkR;hzEcyHAEGjiG{+W( zV3^%BZ-(&X8h|6^MOtA-sw6QRPVG?fT<*|FI=b(X2B;E0ZZ>DJZ!f~=Z zb2k&n0|$!@GAAcvk9HtmME`E43AEm6V!k13ES(}yefy;+4Cb^E;3OTDj)~YJETiUA z5hO7{vjBf>qZl2v;W)YMR9N0j2fewwWYsd7diXOs8b#?kaEG;fch2F%{A~2^^kwV3kM9ovkSNYbKdMELQ}}Edou|x|5!2CILoLC# z!o~wBkmcNz%k-BGmgY@#_#&*Q4`ID{%>N^(EZwAQ{YgEyMRyGYN|AcbB?L}zmcC>F zwW4i63UA^=CawBigZq!~2KSkYYxg-KaKBKYQ7RYH7wknt8w30Yy_-Plz={PSSMu?T z9ALF9!H!(q^M2IEzQMFw9*JCCGm!}I;VuxJ$+W_}!nS`I_HjQ=Bqf)IQ{mUBFdk!E zq~ee+MCsto5?-s0Z{dP*>~wLVbvQ!P{)hnK;C_bZo``uGlicwK`8B)A$BSmrR&AdjwiU5wp~)GL#$;D){KWx} zy#;ng$jbbAd*uBQHs^K6&K}cZ3NWFmS`F8stKNu9NW#rC^#RBMiW--yUQR!cNktCV z#y5m--!sU$&j`{4qwv)szGeE8?)8THKwl1&Eq`qfVz?t?k zhSM^tCm>%Tq@mN#rXt}fkq)E6nJr+E@%A`2R1?&44!gJ`aPKZS83wJrXWG2?7ujP6 z*8{`s=uuO^VcW7R;vHukNRypcOTD&_NgKy+Q%@b#*aR(>&5YUa%K>`p7fXBB55#|m zBx$2MMls#OlWwHUK>>{)2bNsH`?H)3_XwAfI}i<}AEGs^ll1nh9A9%cmdE*~?Hm~o zd69k!$i(Bj$Ls(Ce*JIH#h1YaYWhKtZaqB3 zX{yra>qg;>NX}4DM}!A)Zh)YMh@cM*RSp@tAT##lJeKN*LFJPtp4M%5!tw}+qd^(U z0n75hSjVD@;n&QRote1(4*XFe9WeC*C_{tE(W}}J3`sCuzfXt=R(e;)rsZGGGcj8M zWixH!!=nkFH15mI7FZaA#=%u`4kjX5k2UlYXnAmzXh#C^e4;ZHaIUMG<>CIIgy4w@ zc|u-xRmT3!I3`jxxY9Z+JQ=eh6Cep#!R;8HY!4U!QNa)`Pn{(kQ~tR4@+}%m(96mx zlDU;BBVne8E?zU+5%x<{4sn1oof#NFffgMhNFz?~nu&D#O#;hb6@-?qAh>P=pWLX* znEjEc_v3Ec&@X2TL&K>8TQ2@-0HgGDe{EsRH}bka*LB zw_K4gZU!2RAs7U$5C=q~fljO?TvpT_3+ggcJM@Xx6C*<{ax-3*=|^^U2Apl!R)Knw zjYDheBO_hGP{9Ul{fK1h53sgPP!K~kF>yabX$2{epYjom2d+Ynx)~;{N-RpaA>gN*_O6ItzhYM2{Y^=z<7pl-)g;S<@}A+pl`K}Cv8kioIf8Kr2n5Wx zZLns*DG{|GX`msGON?S$NLe;a3t@pnpK#fFzAzKs?+|(CkL`dN8&+W6@w5tI(l4Y7Ocw5d6;OBA z9&PJn(0X9Yj*9G{9MUp~cHxq+NiN>lvGMImN#^jmYlNp;E)P(dIf*`(es7ck%G{@GwUYFav9 zPzS9>)IvVMbVe!;q)eXy8oJY^9j)b|jN5c5Dk9fn~ z`HU!@ak{XB@BnI+EUB&EY``3pk+XjM>KiN0CA`2{HfYfdNe%~=4c=wNJ9b*g19^u zJVuE8Ok|!H)d;5KFAcKN&y!#%*N&!3wnrYu<1s6z$Iy}DK_x5P;xH(Jv4-_VIKaZD z)mcFv3eSTlcuW_wqszP|JmLoO#KPbBtc%VzR$pO}TcjHCc-*@mr8{NfMCn%DgZ~%YjJ*T}UdBzK>8QXh)|x`cdVL_-|j604p7c!MNS0NUN_I1b_f<{BFNX0jwI6rMC@~zCrGO)dwMw^pi z92j-`Arp!kSPj8i#e{8{d*p~**7at_Id$a>rgb%kd)WpTxa*UthiLl6)YI}@5xs}c zJ)?~<%~P-rFcZ31jGUm!GEEQlC4z?%Dm%pI5)@ZGJi2D!lAf{#CY@cO0nVtQJoCu@ zAd8S(C0W)ue4r*MKER!Y{3iF8nH@1~Q zM^88UOF%amr5nM7F}>e=w$aF-%+&LWbO)UY=n33nGriG`^~O)IJQ7gUgCY3XK}ujs zMj{z{=1dL;h61SR9yNGuRadwCQK%XWWC)ljLjHdi6Kmeucr1^BBW(EtUmzFKi5TXM z208;V93-$1RSE0>Y(}H)Trq4XX$wd3`g z5ZZ^1S|ejxh@~+iwAP2GwbXrpW1&#KRT@ix-G5n6)T09iP~NAH7fhIC;0T>Ou!IcG zNn-{bhU28kBs&?^p)(m6C$_7ov{C`}Q4uhBh4bhNUsG zSG^ve8n9=a4eaYyXtV>0LsXe=Kzf4wKq=pdV=bLK?GmI0b9CAL1BEAMw|SYdI5#ZNcTl#tKxDuoSWt|u+uHU_VZs+}MOcXl$kG<^Za5Fi7#TB-Zs%6Uv2K$JgihP2#y2qlFF?@0384)d(h79y z2OV>4jY1|0T(GuKLU)#2^Xb^O#$88f8E4^mx=@NB+)GKhYA_HB3_a|RCVo3Br1-kE zq^V+PIvt5uI9fof-s}g}VmH#1W$4l^WEZ1qLKd@Ga%ayN2r`@SJQ^rn0KIV724kTO z8U%Dq2o~Z$kf_?Tg0Lj_MbFbRAXVcu?m~WV8bd{r7#EgKHy$<29uGIJWOKA}SwM^K zO#C?rSY8*1rcda%ZuVR5Ma$C$C-sQoR1Xp=QqIu;1M`DcE#?f^Hkk{Bt7Hqs+P;oc zwo(Qviw)-OZ935Am+3$|dM_9^A8|g41FIN}L!Pdvpe0%?J03auR4O>>h}oAYc;=Yw zEs_joMsIJXFI1P72h}x?y)f*+g79dHU=wx*&`+5FAbOLd!-F3b`-S%4!M1@L(HlVa z6eXx9Bkc>x!_pxc!|3o0DE6LF&aqmM_EAuweuDioLh{Ogw1F2w`42j`?=e8R`tVO( ziP`U)arSdDx;zNX9Dbr?()yLWWq+h;!ztvL#?h?>ipdE@UM9D&he#>>9w{;5aJ=Ur zJi8mtq-8ib=F{liy!DPyt{HzNhI5JoWC{d{Fa{yDQ*P@nGlM8}6 z%Y8j-d027hGz=)B%31Z9(9aF`8Ey&&x4O)mLjpRnAPLyFDSA+x4jSO--Xcx&Rx{XN zx~*D%I^Zkxw8F>$`2gHyqqIZ&pO#v~zimVpL$5{^IS6UW*AmruA+{#Zk@B0e`=vp} zGEf8bXGu8!Q}SIFW!j!sW#&D_{eJ^yKQ*mJ>d>D!6QxV7J?6hX?J5D?`xo zjEJeY~EHsp)3(JKFp=bJAl$q5$1$l{>KmB@_qTvU+U>N!XRL2wx^@pCSjI*e$5fUEBF&P;1iA!k z50zbvvNmF-F@o!zxzI-RdW8ESlUb}608`cWW+vq|zBqy_rgB`sA%IzdgOhM1gi?c+ zn@l^zU9yG3iql3W56HkEgW{fBh1yNe986rPZ~XhZ?*=W8|4Ba%QONt?9SiGMH$q4s zQM93L-uT5an@l?VNb!Tx#QZTJ)7H*|p8$ydjbO<)WPoMlN|X9M#QD&l;zAg%ZNsoi zBiI-_jxe6cSi;O=Ki}x8G-`0v@=w_8>i&?7#J@(C!tnS+UhnlE<~ z_>L9iMD2Q1hEPptIKPC_CGf8UpryqsW&tj@1=qNiL6w3m9We zjFG!8$3cwg@T96Za;jJnFd)U2n#b{;2;NmJ%X34t4}DvQ&(vs7RO%7;fB={+;t5AE zW>Mxp0BIRW14YW#(LX28T7XWU;c-~0@R!G{l?qkH!Q?o=Zu7z$g3@gS-GYYZPS)a7 z>KI}(jDSw`}t>V4)A zG(qhHfGzkkr3QY9@UO`#CLQ1>f0ml`W^S(I81@r(#S8>7xhwrKLS2YB+Bk9UV3_K# zhVeK2T5ZK`!1p**B7Bu_O&Nfq=I?aY|M=EfKaP$wDq4(ktyqz%n*9wSgK`CKi2B{h zawlT)4A_@vghT!g9>f+A&)z>m)i*^bkH9Q2kPpx;18H1FXkYh$4|dCBs*>RKKxnX= zA*^W*&_K2*hlDHgfhjO?Pue0VoqIEkbZoNnM*)he1l}CfxgILVjG=&`&T5#KZ%cr= zjg~VhOOFOUS;xTIv(KlfbZPcI_$B#o-vbN1Xgls>*h~8qh&SpQE21Mh3L7vjKvP4f zHnAkzJ*0&{(xf%mY+1%E^(}XQZv>X1unstIhDtdx2!}!Vf-#a06-ZDL%#P>D+Xe=F zW-_F}K4Vm@bBq1KZ|oEY=2sWKPs@)xQ3|7lX~`c*D9M#kJai@jK>XZUEgxho$>cWt zIt0IgjWC_*#_~Kl3Hb~0+VerY<`1ubHu^jIQNV8VX=?I}sILsG486vL+SFcgpSdTI zEoaP8fbRW-;w-9Wl6I77@bsd{F-u8rk_ zA{XKt@6%uj$O~+$<*#e12HObPwl#J#3!`T_<$o$lS-_%n^{Lxy8yXoNRJzKlN(?zo0zAJXe?z%nvNR0e!N2~7ZS4m6z_w8358VmJHN zdS`i9?Pz-Wawy4=GHa^@pwm9f#2kF0YE<5K60{NqoYg||j)1%%y^Qn~2=-my%B5w083UD=#__Fl zsGK9?)h&t&@hi}B$%DBH%{(adq^s&Ho0 z*T;v5fHNG@84#^WQhhT_#IBeY)pkG~K#n%`9Xj>OD}Lvg_K%CS*pB+uUvm4@KXB#v zM+{FlMSKmh{c zi1QGly7p&s!03L~0PJUtT)*Hktk1mGTk;3U{Fj{d$2X87FedPd?@qO_mY+Kc63#lk zt;B3AI=!3}x(0ZJjbPJGgp@&6Bsaldv1r`iUMqilJMERA#M9oy*qy5}h%dUXfT17= zh~QG(2i6>4060frHi~Vdvb!EIww}<|e%pAmjLsTJ;UIDhxIduj!7t``0@N=HW|6dR8-gnABM@)v7W%NTz_$U9axpd0jo&{y2&zMYZ(EtpWjLLqGc zCgIET_*`O-!3Z5-!!s5#Q2El+9_L!Fa6O|%XMT%bI}ng)u|Q_VCyJoF(=q;L4uiX( zUIh}SE1nh=4ERW!WW1et?wW<=saeos4mFEuq;-cL*+y_9!FQft)^ZR9NlMY;P>}_M zVTCF=Q3RK86O^7=S}I@A|MW2vrKviC64bYK@KBJb2h)$Mpn>=(?j2w+$?8PhwjA4S zZdI3W49LLW+XL`V>-_CGI9=WVl?gJl`#(xj8Gy2A87ZRA2!ydqp zH5xNObiTlIN928vWLcg%nuxZ;$rc}M3) zV3E;^OYgo;`RcH(GEVVSBJ8nWUE{W=_mq@oM!`!#(9Je~(eei3U8XU% zCB3B$$~^#L`9aE*Knhoz@^JN{8$P7t1)iy11lk9tQG`(CLadLHCEOR{sQCbaehQx=_QBd%c?~j8$(z z=^XL~2I&M~y2IJl&vv)Ho^?YvmF7PB(TcYxs8&oKWrUu``vt5)vBrIlkpPH+u; z3{G)oM(%@|C0|k=>26}#hk?=o${}RV_B{qEc0Q$s)7yCW-SwO9_^jm#kA9`$k*4q4 zeJrhH6B?o9v|uNT2iEpT8q`Y$XnU>ebL<Df};7%Gz;ZUWOc6#CG_>Y`0Ssl|Xgm=O+N&al`1ejIM^xmen@&3~ERd0+tF@ zo?%xiP6#BqcHFn2l;bMZNpW(_!e(>f}whykXydg8UR3OVZjfyTkxm|)L8q&#-E|3|b!K5Nc!wY-}dvlLV3r8-%mt4{UoZ zJ8*tHy5_Vakafh_b3X6KZz3vnuC1{34H_9u>@q$UvOc8ltMmj(jqyM>RHvC zy??9+$eA#dM2qI>hC}=FZZvCoB*6@C2!IaoSZq0a3vITNY_J2;J$e4)#C=iD9^44i?PVEP%rp~S>W>z`Y4sa^8%oIX41enJECx=G zwKg_gIb5HTWrYfWZm6W;zXGnl(eYnaX!O%6HpflH-P!q65XFq^OR=$^@JuTs?gwk= zFI1J&83}U$v6-!S{frea3@kMV7cRg!0Cc>PK^udmNWVec-<34W2u-@{!_eBs(TK#& zkfQ4RNFK2Dhl2l9{PlTx2k+GAG+-OQLOtoZ5$ZQ^tH$VsDT=>EVx&u70%Um&4kAhw zP#bV7{5WGHd;zbX>@74osM?l(4ix)2-g;!^Rt{wuVTsftCcpd`V;143@31L@WkbeF zYa!Jbz90wK5IfSrO>sHc33sFQqDS#`=*nMOGb}@rti@5_Ya({0(uE|+nIfVN0NR@6 zjC>lwEb4klCK2yQYg@ny_3EIFC?Gs@_Ibl(So$gK8Q>GWgk|AUx&+5WZ-A%jF!%h* zX?Zgt=cE9G2w8FfsAy>-2r{4O?%i)4U6xV9XTa;nRhfaat|;`8!}OjnFT}$&AE>k`fl7 zFR`*d@};owpR=&?-AyTbpb1jo zvZ-i!H@oXsuRDZn(NPL%>twZNgmW_dzIn9QX_^>!d*mLP0fzX+qrnz-reMnlw-$1V z?+jtwDh@1TGR#ojQo4X$v=0~K9T+&~PHTq`MUWuIheBaOqSYe~Rca9}5r*5)MM&g+ zV*y`o?EozgIzpvE_$2Od&Nj~>lueRHlrR@3Y0+3VX|x~NkdJG`H7jCUmD|M9_X4~?gJeH zN|yU>rVzzLciyA7POTQ4oQmuf=;Jx86)=Lrq@;EmO{?1WUCy_B=Z)B<9rjD;Bv=c% zD7aIC9fTBQhk=tW?n^;^nHKRi)#sB|*{o*}R2y&at#kh69N7;*p~3PD(g(d=1WMaD zmlGNR@Np2>IVG%ZtRiC@N6Z5+&unE|71s0ZNHgo5sc0E=IE-A3G2o$^h2Bv^4e1c~ zZs=EyqcA%R7D$+L&|(gF8+zdQ+S2BI!)3lm`|EJ_K^(;Nk9m!odgC-wmxpdB&%r|+ z2UujyJu>tj;Kf8+BN<@`!Qt6ggt>P$7PX9ffuUibVzd{Lb^~L<$T7O-2AxC0>bB0b z)IL#&DG@5#aSo0Dr!7sd|DCj_J4a%9a-*Jxl&*0@F@gB|_ zIB{cR8hjm30|*QZV)q*+Gr#ctc?Q0p8>R+46qJzSGqj%=vu9flIX4AF8XGYKnu6L7 z*gbjXgiRAKD4~mAo=vSAOI(lPdkxT2H^c`lcg9h2AMq5K^B!?8fGsV)3RI2WShuJ^{ zQiC`ZSeD;>f_vu1@}Ove7g@Z5TL2}@G_j9%^UQ;Nv~p=}qyQ?}aG}GpX`mP*7AEt! zbM#!_u7T@z4`q3re1T0x5jZHq&`d(LCj?_rGQbfLxIl}-K1n$}abtZV-b zJ(&LIjVYfgM$n*CqoZcuNEFtf6^`h$m*jRGPa4^Z6Ma$2bBURwzACsxa39j{1Euqv zzm_{)X6eU(Gv)Daz~QA@Aq0|am?sY8_Nq-Cqeimm8g!IR8`8a;T})HuqW+DRP5p8+ zW*NPlkI1OLT7@@-Tr!}%fad@iM({AxwHlGtJ3#4GVUiQ1G1R88hqhSSn~JDQQ@{^DXj<@Qv0zN9&$SXX1tL+~Av5X$K zGK=U@pvR1aGRTkUCjs*GToW`*B-DQXjK)!+cd|hDn=vGw#_$B56X5(Msy_0WH85gx z(NxWfg#|{k6CiI&6P%`#2W#OdXRR|*k@K&J71txmic2gG*81C}ud%6-qyast zO=1F&)CQ?BF|K*Ua;?LZ@wUUWCkB$Jj)9|4BvGt-PMy9t!uV_i@VSAo$rPlP zuOK0a=B5jwL=n|iV9Kmtfwj2qfelJ19kPE4<6r1&Z}SgM{-2lY#XMj@)d7zQ#SA=m7`!U>G4e!2 z0fh!v(loVj!GQw#6uUzX?k617xj1ObHgpp@~aFxQ-ccZFg;vvozLBT|45w3}; zU~z^hfN{rywNGF2J@cyf?Zh%L#GAP5bXCEmBS&Ie)*ZHVwjt_Hmt7-*@dbJV9ThF# z0MjgH(ddknx7B-YUs%P~KmHOS%*>$I2ra_l5kYA7cVhHNn=KOK8B%NHsm-SV*G3x% z=@FuI^>(;u-%`m(ARvV+-^4O;SeLpUWeEJ8@OuFg3<@W-Ll*med9bGYEJxp^+0v{y zL3vA&rdv536C}5lkG>-LZwJl4?W+|u8qnFOnJDUjPp)A*V@X0ddgTLhCW%NEykj`@ z#*5BdT#Ka(SN@BG450)n4x4tU(45VY?{bTeo!DMEuh(F-{Kukgj$ff&#JPTlKU zR4RW&rGEM(J0D|UU>XsQR(>;dT?eFFy8Xu!46M8Y6;&fGoE?^z2&znO#r4KYVnKPU z9%|`lKsVf1Fmj@3Awq6dnh_Rpl;ae$qZuw0jV%lXyB$kivPZoDNeSggjJBU$v^j6U z=$yV0w9jH_M({VQ6F6Qgnb#9B!?d{TQO*AOJ?-*wNVsJ(E0qdHQS2_61aI zfm1|xzZ~!{fGf~1?R=`@wt=PI{NsZZh4XlY{I*fnFk6?o5Wq5AQQA{=&{+PfsSTDt z#%(($@Nf!2Qe!NW&lArk7l7A=7Uu_!*eJ2z7?SXRW;$=>4uIFs2?x-LyOX8m>12^{ z2TQ=PJ4~&53com}ofylS=95DQ&31r@x6nrpm3#X-7xPq^89miP@)tfV4Dvwau)}u1 zjpz#?*ESjahg<1H+>4$SUSts#6`|wh07M7@Ds&YL+@1Kfy@Qf|#GNQ>8HqAHrtt42 z@&HD!J9ff!QBWI-(cp1o=1D4?t&EYPAHS)WXzf+|kv{?C^{Z}Ud2mVe>0LQ4n*%IB zZ^#71%rlHU>Htk2A1m23l1m#d?xC9CD2Fm3U*Xx*B-CCbPc07vk*t3o6VB!EM&~Zz zMz%I_M1pg*Po`?w3fET0bQkgslb{)9FK-%#O0tn|D{gDA5Qndc=eH&JIn-M#q1f^Ly?`V2IhPjl_K7JZ)7f{$vhT>u^0XD1<3y9X1GvHVivlF@QrQX-Jaa^`j~5w2 zxwP7REynFivxiX(E_vha#8tiwPPB}wi9wR5eKco5p7J(>n$1)nV=%CL)I-7hYXPPn zuG>bK6$0TRpPjZ%dmqYtGay+81v4FibVt2mg`O1cZ6u(unF5W=N)v%$m4uK8>X=)0SQAArgfZKzP8q-p; zw7o2)-uAnh^JSLEGWG&IQ#WpaDO=ZdVo$q{8zju>g)=9>un|}Y@Pp{ba2qs?t8*-+V~EmQ3Yks^nxkPtWRM&@!oAd4`NnXc%tuCOq*tP(Dg_H~=j z#8v;&7Y$U)GXn3&Q{J-po6bHH)M%U(_9kc<+^1*YCa?;Cgl)`Dct+Zs!|I3aH*>#G zoBQ(@SS_D)fp+R&`sm#zRMp#;9RkvOV$x;8Wx$Ft-LkqB-25o3;Qn-5M$+wmQ>|_i zD9b3K;HZU@jiT%ZFI#E59^1;!Kwil?Wc{O%XvrXwiX22WQZ-QZ2sa6{c0+*i?}c;z=zTU#jg|26mNc zW~jECEyS3q9bG$40=lCJ?sd?MAy=@uxvJ&8ddx|AAv#NgDaf1SgI&Z89mOcox*_Pl z2NH+xi72v;H*S9)i``~L36RSH$LO9-fhD7CsUgdU+5?dTK;>`D#_@u^IVO94ET(h# z?GSE(q7L)MJ~o={d4wbQ)lpkv)rGI{wDI~lMyK7ryZBm+r>xX2U)~}tqllnK5mt1h zhoId&3J+OS?j!`Sm^>jb;-l*;m0X`;cwpLj2|(X>{f#OrK;N(Mc*LUdWi4T>5qeJ3 zLNdn<675F{*HLiV@@-Zvql^ycX>JYXA0}^O=Jh){()5+Q&XDXTLq0a8oy3Sy!bu3} z_;&o&5q}K0H=H7HlG-4)gFEnYCb*`Mmh^74TLa|M^2s~4<4-6w?r@UyuoLZOkv}%S z0P4}>%`-=uxICq7lr=_=hsWx$2QJX-+TZ-gudV{igQ!nA5>Q$wz*0T(h~D8wtap1e zuCUc%D1);#8D5P~3BVocwSolC)3B1}>y@mqbh45Gb_L1_CKs8?q;nGHG4#8dl|UH@ z_einn_>vv)oMM1$7clgFZHfH0?fh!Ar_*WYSm!9@2*@SE0MPchd3M0!IKC;Gyd9F| zG#*TUzK@&`y>M)ZRA@h2JLTI&pd1&0pGpQ>JOqP2(28i-q55*r^8j9KovU^R;v5E} zh$j+sp=gBmN6Xv$F~2qZZ!wwnk@osoot{Ty1M4RCG$bQ!wSGnp+&>wejS?~{AT*KC zwdiFGgG;sB3tDq@oPQ@dcqOh#4tg)FV=Rv67_=!!Gxc0Y3uvJI&+Uxo8UkIvgw+2C zImRfsEP2AT0A0SfFWg=?&xK`Rgjzld6*0!+w{GlX=AjoPcfg1FXYRr<_0bK5I$AWj z8c<4bEX1$&v>e(n2(NjxmY?)Fho?@5$_b7a2Zex&;Udn^k-(UPCcPoQ=?Or~MwT%q z*}6DAKgHyuz8#IfoCLHCVEdwB?qmhMQ?gi2Wes0h24eQpt>J(Nh|{Wze9+Jlw^pwX z(t=pY2=W=4zr}g$5$6R--;bOaUzL6~J!MbTOeaH19jsK1=S}M&YS|hdJG#3%mVO9Z z;KB3Ui}w}1&IPTPZf``oj7-6>d<@GQuooCXn(Y*vw>2ZN*Tt1Pm!3Mhn0r%6QxuG*%*?9(+o6Vd((I0YYWbG zCyOk@o{=9NNSJ0s{>12L)ng-%ZL!`ERh+fV1Ga57n}QS9(Q;RwTAWzwGvzm&UYqB> zp0zwEW@Q^aW>2>G0RaG@`%GmatJ7Xab2a=EJr7i@<|-%-Tg)1e#6n&K-0E8|wofNzKDlS!BmIP}N(s2QQT!X}ZL(g_~Z9UA~S{uE;g9`1EDu>6dqQ<+D9MJ0*B zkjM{4qXjW)w-0RD8+dlQs(=6xMU0fOfR3?9H89NmWPSm)zK?{KXCizgFd26#S);pY zd_;u3g0&skO!AeON5V!16CQjfWQRcXfaDz7Z{_e zqzR^P18UvF36kj-ZoOcWT-lPGFwa-ozf4GZB6-J6D`0tYwGk4)%dw6_<;J^%9^`m1 z?#71%XaYZUUP`elT*%8~6$R*W!CQV8k?0WIf)!eZ(+(GN%sQ}U$nGQAS*&3=4wz48 z(nHATLUTNKOoC3Cg_dp!_2T{;86gU|OF4xCF1n2Kyv_l`-7KQ*EWyMVQw6a0u~%b= zCFtAZh?s4Y7!6S+j|=4+1XoC0o)Xbth+dW-8ORk08(jkwV`zPk;DRliyYPpMS;B4y zp*M%7v_GGV%z?k!vEX^fg@z2)Rl#U^RI1Ie52;OZY3M{mK7$xhh%Bb1CcAuxdJ5#U z41zpiu^t7k?QMtiosr%GW9v(kbFWiI={@?1QcQu2T{i6KE3BVTZ=$ik?E@6vwSZ{o zk7wM%@;T5I?*S(G3^Jb%l@vVJR8UI?r$ry%xoUBNNVkoE;1M~kLu{=o)ODj-lU0_m zAptPwB&KgG%N8b;?>UTH3b&+E&!0I=iy>q~8x&PkIDMl>h1)&>MYMqwR-(b0O6$S3PusFEjC;{iI z_F;%&(uD51vW<7f0z&NStY{fHd+H+>QDn&$h?y46t{*d^?63)%JVrrG=?Fob(AY8v zoWVhIr{S$hiss6t^`_;A?{j=ID=0<;3~_`E2|CZ@`m$(94iHDN+d3fA1;XD2kBcb1 zK(ej=wsN)&!Ph*j93y4QB#(kh2+?B2l|r9{0c=A#2$7K#l3OU&;6MXEK2(Pq0_xT6 zfUd{)L~k*-xDy5~KYpdtsEYxz5Pf7c{X-uj`}uMprp-0tvc?Gxc?FLzgRieNmcT&+ zL=WkyP}{!{wJblvth5R};<3UP)c4)e(UIUHO(DkI(qFz*6uMY$rLYjD@4zcVRy4nv z!TRi9$YL#zw$yREhtZZG9D`|#1}rneVRg>#Cd-ku$Kr)Nn_D~7Tj_`BF){ym_f_V) z8F(y%fyeC}RtivCfrQdr4Yo3kW&aFdlJ!X8GP~zG2dQzaikJs-ZY+}azN5Y^>HfMX zQK}ByV7Iy-$H9;&XKi?Cn;Ogon2*re08{T*p`$tjTTOp5vth_t-b~1JLpE5Rsx~v7 z54AP!NYmYF@RJuF0ogdNL1B9gsAxm59bJcpc(x_rXr-_DHr7!2aW|M)p0=|`|8Cf^ z=_~Rbj5z|Kg(WaiVlIwxINg(1G4CL8$l*O<3!Vl`iVVEHy*0jtMCuU|`RVC+Oj&m~ zjM4DrFm*w~g(}(-4gt4^DMtF1AmoN-UuQ~*Vy#v8pPgSW)?3m>%jo=4C;I^SN`hU> z7(NBAa}%l1BOH)|eF8T$EZ>vCqa@w7!3s>yKHu$yUB9x0T~E^$vu5AR4CS%t=pOFr z*KMB@i0j2FWc8mvqyNoE*6%_v0IQLp6M_Y7?)a?tr%-f)ChrS zMlXd_s@t7QMNa4@P&op|hB;UR`%Q-&xTh*mrWNhobX_-agJpzh7&a|}Z-L=J`h}22 z=oD1@R?yU41i>J&GCH?+E9}4=ynSvocSGq)KU3J=?{lf8&n0i$ZCAd4r;xa;>_}mA za4>azAT%6zIT1_@T?=2`U|o zK{ID32!T%5bJGXHcww%P+=1WAXo8Q11$2K~sNVDomX6g(HHFs^5=}G^fQ#uzg}@iK z+W3rT2&@OuM0$d7HG`56{b%QpVd-r0Jf0U89+nF)3Q8f^EJ4d0BJbz=p`R&1OD@9 zuVL+}yC{s#t1*ZLb@?35dCSgpx+-e;qCA1u`!4cXo@8DpFeZev*_aLS_7>jN_D?S8 zn`3kua91=J+>u7+EQc$}=ISOuV{W&eB`!h>H|kf*GoyrJbl4mybr`+tNfFcwV;MaI zjFUpn#DE0KUR>@4*^z~!IJBPjW@^@X_h4C`c?eoAhl$}I1`9PR#?Bf$+iekGA4_xz zCj`@q0ec8`4V;}Zz)VOHyt9{L?)G05b}@JR51!3%{sv}tlc94Zj}Xs6O$$T-wIEAG(T!bAnQRss+oJTW7>7D{Am#YWr+ z%^+*Q1sdQVZ3dK(%-z5aIyU2q2YJ{Y z^fLgCfEkPyCAK_e0FmCguj;%biC6s*gF!ks#2q}3@b!w>Bw#xr5{m;En*QkQ@V1%E zhrdI%BU)>8Z1TJF8D8(d%82>79CBcLX&ut>Bu%Q}eyHaGtQ~($JRlqmDveNN@!;bD zA96TdM@3^ZKY4mP>#KaV+}3fzqGK9y>09wXJ*g-+(kduS{|9rhmk|ep3Y&ovQBoY3fVNnfw18@IpX&h#; z2sDsQ^GnJOR^YD#Ga0O8+aCkA0I*}v1p|5TX^@!u;e&k0&(I^TK|Qof>=){TO4GM!EIk&z&Ta81aL5BQW@3F;Y0vMro=jb2I_$^hiv%RW>o%j0U^}gr@uk^`PTZfj5$O# zV-2bADU(oZtS~3@wAHE{IGD@^lP=*IfvqqZRE0ob&XianuYX_^WU;&MFlXsYjG~#f zC!Hscn=JkW!)`V&0@piq?({|flWh)x!$`KJR6ES+Y%m<@7-`3Q7V_be=75iYwywR zAJR_f&!gq;(pK*IRBBP-prh-`jZ#8FeQsKs zo-5Y1(^eFt**af!{u;^+a!FTh{f5O1^!Q{@kMUNV0V7>x=_Ia*m+~j%SfdXU_Dsl0mFYxXuDY%$gIz_&}&_-t0un{1gIqzK(vo=C^vM6AX!qHmT zJw(%P!<^waeR@W@?A?x-vovDnJoGIRkrfbL@#ponpcaKd-gB$NcLF`V{iznZet!r-9v9%$`+`H$U`_45P91ZBj*+(3xJ%4 z%`j*uH3N3;d`q6^yzT5P?I!QlI(IVXFx_##g2CxqszwhzDcYk(xGe{gHb{7sE3zQk z5^$UQ^i%gOm)M_YGRq}K@vOs89StPI17HMOka6yIa#Fn_VbF#w8x}x?>1=i99-kuw z79gAVJ7Zjw-|MuT+4osebM*X4S)1@}(%stjzmQvybaRAN>AI;40EBf_w;SiMps;A? zr~o#7YLWMVFyChd%iovb#JyQirgX~mJxyWEK%}GLvYR*WJ%z-POEC4Y%T>^dkU%C< zj%Vga?{|JA%O77MMvcsxsf^AG(AE9-7macSAy*Q~OR3^q7bYm>+EnQlqMA58*s}Ng z&_-8qi_k}2KQdts6?dWM+b~MfP*rWpzkrr}!LVRO-??R?y(57)Ik=8X7@^xFRUqAe z2VQ*RsHziqYMd7IN)hOwUs&C+@J){mNhxF5!26?DSAdm=#RA;HBq-xRVU-a2qMB~f2t=&O61494~u*kFD!Lp zQ6+8*2lQ_pc)$dHQ}u#u0Q4A_r((@1c3$2-%y9R4CY{&7PYC{uo?k|2p4eaR<>;RuquiedO~=T!Q22q^Sp4*CXe_*j*0#0b60FI1J}|2JuPCV`-}xbOa9okg+5{ zCy*Jd6|I<+*OYhGeNW=hi9h0oIV^>PWQYVg3ET19np#sb*Ja-vVO<2*t-)9UWd)k? z>|3C!m#t|G^zjNJh;~N-=?0tmIVKRs`(Z#oAeqN_AD4)4?&@hq`n-Pi^&GeqJ6&ARpznLMRk<2*-q7yK+;wD2r%&pyR>6B4>Ql5wIfK7Vl5 z^YYWItE~h&*Uiv|{F$s_bcXpIPIYsf&&qcyQjp9lKs2Lge20PrQj=oAxxVb_UNi4L zvmffpWP}_ucJ8E`|6*zHOVzrsbu8^r$q$M#GTj##lcgFtrAsj@NLmpc;`}yTt$Np-lnu(YW{S(XEE$hmll*=D`~R zT6R*!&Ns>P_(Xb>oAO}+X*9}xiWFMQt20=6(OdT~RAX{~9VX=(ONHbY4i4~@7SUx( zSuxCx)?3HJ=6%!U&ocD*naO(`iD*q_LvT}AFC+OAbO)zaF}gnD2_fniW=@%KN~b^Mz5 zXKw2u<)%p%7+BSqMw};A)Zd;XVR0|TaF7*5`s1r)OXI&WBPns_IRX0HOEUn-H;1X6; zNTpjT=WZ{P{1EvL;Gv@NR|%m;M>Ggp&@6X@1_zS@vhFz+O7$K{u=GG;$6y2CpU5uf zq)yw90xojiKEMhD;yA5M=|>YqS@-!)L(b9YYRNzA)_o7lS?Yi|FPe4L@+c!FV-V=7 zTuFu&^CU_IXZAr;;_>4?t(G5~=Eh#r=*~Y|UFYI`E5KPAbx%iNM4$yv^-?6C_I(6F z6b7WJ1q%pTAeRd)UBJU z2!SIkPN5UvPn|#D3nI!t%th*Q75p?&wno@F$`ryOw_4AHkF9jkYnR1t`FQj=;dc@R$T4aZ55uBF&LNWv(r zDrSj#A%&vunsOt6eo6)?udeWev-I(>^z)JNB7|QE6^iFAaJCtP>oHwUg05-@rz}ml zS_--WBSVG7ac)@*#9fSjY-JB;KdhH##n;XwXiLM*o!5_b^R_cQ}Igr%OONj6p6cg~xL zK$r*vrn}E_k$>o>>7J$EMXu^b-|*t&?BC}ZcbAb{8d;STms}q#H%4VcCk+p^g9Fb= zR+joY>mNyRDCO=E?OA$LlHbB_6jm(JYX()}GHRJf&qbtsv#q?FV1}+JcR#`#C)L?T zP~QODc*64LeTQV}8(c%R=T>V{`9bf$Mob!(j{)78y*Mb=coci>()p+Q3_x9*ymr31qFi1n8&%5Ghz&a+m0GeguraCy+m2mlcr?}gw=|1}cY&y#># zE=gqhR^MOSAyGt{5SR4(!xpmx0za_KI+Ia}qB4(D8Cu(0L3DD7E##q9kLk;nyRE1V z{}egv3j1eBJ#>Zr-~T}ZXt+wJO2!rhSKi=)QwTx}mUFauHj0insRfRYa>kiZNkhWy zSeR=+&x8+}t^Gl>fyPc|_VcH40YaE$fCHw~Q;42#44X&tZ{bv|4i1h7wwuFL38Dp5 z!*}=uJ(SxS#L{46d?woMB;2aJcZ}?#c}dJSoqSGS4@WU;fQrE{864`VZkrIVk_G+? zfAhaz{EeenNa=GTD2fS+!}Db%Thee|DgL`4Byux>YJF(-m1|TIki!NO{lS_2?01(> zrB>#jX}zaCG_NvOwMO(*3?Arf4#Xi232biUktz#a_*Ai}n#7XDygm1uO55$(@BVm` zSQ?0>ZbdwhNGk^g>*yX5I&YY}REY~DTLvMrq+`Qiuwz7e1>EgS?4M(>dhZD=y(gU8 zU=5xUIu-$_h%~UhxkC#*7i6vo+m=Z>oT$#uWhHqy2MQ%4z~tFp?LCPrw>gSsC@X7= zAR>}|8p6h*M!T)d&TE8%x6N6QeH|9LTyX{vJF|&?69fEbbFJJ09+p0(aTAGQfe{N| zFkh=5ghHo3@k(x5S3j))$Yt&^l#M0=6C!rLFDP!4A#Qz>^6vf`%Wyx8>QDhNq-;8f z=Ub?`I0+R{hsDCGI*5J)^lpdlVL&yrS|oT#eV&wS&bbf9RNKP;_df`rtS`?(wj&`v zZ-8ZXph5YR&o{*Nq{mUJ(kV+;P|Ya%*=GX+ zG33fQ6$ZQ6;SWlaY%lu5HpTOw3z4WBKERVc{|PV5f5HnhBYEe)$w|;~ODm?_BXZG# zC|u4Dbw>dvg-FNrVD+&`2!@*usm)U+knbtfHl6JML9pAtJT$UlHPCCA=3LLxSi>F( zUDTfd;~)sj8@gNZ;2z^1Zz{5Z^o~KOj-~l4;ZZEj|H;>G4MN9Ogz1Y5QkMzg>=G(! z*hA5vdl;99{zm;pcU0Y;t^3V+BeKS#&zAEyGHm`L!+^E=3)})uEy+nr^PYvG@9hP?u3jyE;NS>wlC$h;rfv8YPm3fS|(oPQ)`qXM`MbC$` z5!UbQN)q*z=Tw;dKawjbFrH;}x?#^OgH}iL#97~D7^4@Yeo98Ni$J3?oy(-Ex2nkl z0tNJR-ooTV?jje`ym~cx`UkY!%~{6BZ4W5(HQM($b=ER+m@NSW0l73QGSG>^m5;yB zuinm4q4^`#4@#(~S5PtRf4zc=VJEjME2M^!6n<_)yhCcQpEF`L6}tgTAxv^z5w2`d z&13AshK;}r%kQ~xKWB1#+}(GUJ}-eVhVX)sB=8@M2~JSMqsRDrblC@KcMhjEFr6XB z;+Oab!%o2_^Lei&$mrS{SVpMgxms_xxkTKbl9TTpCHhipy?2TuD<&omXh16y27&>H zM7&lyCb&r7R3hMcf44}%^MW|hjEBO|i^dYRezyixJ0n#c302xHxVN0|3_ll1e~#G^ z<(b4g_UX4XXnn4AW%*hMufqNdEMjMa1$R*cM1z8a#tDVo*{=0M@HGuJGt}#yriWFe zw~F06|JYNx!ijj9c+c_=smpF=q-%>S3fKa3&`<-EDzyTgJv|Gw6)bmJp~Z(bw|S5@ zXBhpyCqr=GGh&vedGdF82(i^m0rgB1+-bAGg&6{F$t=6+;F`oBvsu=wBQ@b_j97LnnZpj?R{7?)TIpNgfU3Xf?!f2_*`eXbM#lpQ9t!ii0&^Afs!npeUaT?4i2p z&p>LZZqj^~_uMFXhLTm6RO?C+&NN9v^ou#>6zw6*Wkz_-fl?sANuq=F{C0|s`$EMs z&ZuIHBO`yL-b)?D>`6T@dP67yRM6%a-h8r>PQ^QfZeUA0w@$K0g$aA&Ec@uUc88^< zuD>+*a}b@K0P`^PEmsRnAl2i>FcRu(WNissi0t;3p$oq+@a4Jrq<1<6`^w)Vf~5yy zxUzFyk3^j>S;aXvO~Gu8PrD@BvKKzpou3RZmLZx+hRPy9!XS9XAeSfY*b;-7+H#^ve`P?;j@Sv$x#cXO_Q9Z8NR({3Ph=+zroP zwM~~nGDDiwX5+9?brjuckri0W1H6r2RVd{B?!S{uaoyy|rEn4fA-@s*gC4x2O^EPi zrV6|wn}F@#2n1N)rdkJyQVnZ^6VaeaG7)tCXn-tfyz#tMaSh}3yw#vic zowRtt5;lh#(p5^Jgz?xL7J}A_?4`=x_Z(664gI(WF)TfZ0f@Lw89)DyQ$reuO#xjgL1$sLqZ(AyFA?{MrOz`dFZB5xb-BlyYC>xhKl3WuyI~iC=2yEHY>gn;2t4K>7J8<6^XUmnvr&r3#^vwnPskj3C#r?=Az0aA|f0 zoHb*y(f;$)LS%0((gKK?+{8(&w}q7h?mnJ+aF+iA*gZliKv zw$Yu?BVxI`-b4+la99oD@A+>$B~0A}`aheq3H12YN{N@df!@NNi?lM145e2Uoj_sV z)D51@0OD9vfNFrMPKDkAOMZ6TzS{Q|+IKko&ch;WV~R*aEpDkJpY{|^T%ts!Trz|p zTt|G%NC#&$Ne(oDxzA}#y=Sv5e}!{xKdqpk9(wJ21U>`l#H`K~EdLM_OXbq04eb(x za5J_0G@lsH^I@G==^n_OrJApeZ*)hPpDx7^KZf;5RoFD$na{80naE&j1{)7G;4r78 zLL^{EqtXP!0?$~gk85D|2exAE}@#ff$Znon!LBKQ-7TNf-Ka~OseuV;Vzf%<(cu#6;z zKQnghCP=3^A1xU`FvXaTdr`7e>YUu9$DmGO*K}%4qW{xqdY^-KzgsHvV)%7(E=O-#-R4B0U(VmoFLeG$qbnIiplSlmV-hmG%?)$_mmEI3=cOgzp(HH56R6Tx z%N9QpuNEU-H($GCISj|SE?U8+VDd$X5tqFkG!Q101c7I!rR!}w_+&PnWe7D|4-4ob zVwz7teE$&muB_}Js1-HOpX{F$8;tEO^2cp}+bIqc3UncsKV`pKBAe~E9hp(oKv z!+A0djzF(h6%_=ONV|~(FCbVZv;1^5llP|n{zKk z2&OJUCoQVLiByV`hqhfOM_1F`z$`|p`0f!p@svFG{2I3U*hv zhBjMV)ft2Zf-8%ORH1Qn^hG`Gv~DK$B$b}3>&YuzpJJY_y{vZ>{ zIHk$e)_3eT7iSt{Hy*~;P0Ff>8{wZ)V7wL1AL>tlhyFt{CWrw;#l`@Np(oB|8w86k zV?*qbp!$+NtzI_@N`sLHJt*>8&mHD%h8mwb@hpEvr1K}6)j@;cd>>qbEMrfkB`)i} zX_Qnq81V`=v(J`UC$^dD>%>_2Jel(NBy!|o*a;Z={v&vx*=y|o*dx`59}sLb3K(** z2QV>95YDUI4Tl3`g|t!FwDPvja;vxl$w0T@CUBrGMcDi6d}0s!g{nc zM?b&Ql%hk=u~NmjQ}S*+t!ePuKVG0_v9|(SCD(FCs5*MYn7OE>7 zD#Z;1wkX*28#Apj`JatxYfMhUBgPet5AeZ3US|GldURT-%E7eh#_d@cQd^u_b9Qch=1C`)~DaSe(mnPW9&QR743358qx|#dWQFE`Zt0~mG}ZkND1*(SH9my&c?Oj z2{3K<5y{d=q-ZJH^N5{=!Z#HbXV@{cxpQ6?Z2f7%oS%M-Vv7|Gf%4j@Zo8J%LPZFqWW`v+SYIpkF( zY<~jK&dpmv*{tCnAM-S}S)KQDnbNJ6#1xH0J77dTDPxT|Md;4rF&;8}Kg)!5_xr0; z$2P=T?U%ttb-64lUS46>1s1&~vCgT{v^ycQ=+{Kv^E2 zkS#wefC(HY*1x0EI3N4@Kk?4SSS%xRhcQB0j(frQ9m=T`u(tj~*}6Ve>sbbJrF-r| zfqHI$eDKX30v-aQf6=U_X`?D!5iI3s9+!KL%prMHIn&Qq_{3Xa=luH-u#ArYoQmfN zB;F!C2YE4kOegB0t*#vnOal!TC`B$S*quE*W7;6SX>0FBLjREO`wx82FxQ%O=q^b2 z3JCt{=m*Af9keXRpIz>_q9ERL+cVFNA_90?MtS!n6k=n$%Y+eS!hcQeJGGR_S>2Lm zArCSK1{|P*`mi3xV4OVD3AQT5&e3&*GLqf^KA$p)aSKIQ{<@3N=Oz~mvdc>1Tj)U` ze}v#1nV*cpI?tSP)AgY>$LXAX8{Hu%6U_Ce^J9FSAH&!9rjf6XN3?ATYcdkmaHN63 z;S)wCr6S(H=sTqbu;VbZitj& zG?#Adny5L>3mGwbNG`=hJC{ltz;AW!r%b}1=|qq`taPu00Kpa6kTJa3!CgX339!Qg zPBbKqtA_Y>j#=UXJfBhhCnd7hegmCZ#-7|FPZ*RpJR?nztlu6~jf*fvWE_4Ihz;bb ziAbu3E?3FI zo8_QgT?eHQ!_cfx-PXqe#L^BRUiIz=dQci> z+%6E*!7N4_z=xn7ogT70Il2@WHTw>9?1p&_>w!S#?QDvs7`+XU?45E-oAAEDo@LOi zv0(pkVnT+Uh$Su4Jm+X3>$wxg<9eo0At#XAu(h|*NFz(0QM85ZC!%lBnZ7=Vv-~Ma zheuENB0&U>rI=Y4A*Xtw$+}PdY!z??b-@%ApkXisau;2ii{6H`@g!=TkNLKIzmIL0sLi4S7M9HR$FSB@>e)3gu%U84s>5W+{_>ACaYf8 z`z1m9mQHc%oWOH579OxHZpLZ z$tT`jGC)RPL<$jE6lg?;=nE29g*G?2Y02^`kVkrU&kjc~_W4w{&+jmse~JIFG6mj* zBZ{qPLB(_PJ(FWg!W*s(vP^`>;N1z$p8X#!u2eqgqu(wGFvva(g4#su zK3h$;D*=U=P)x|*y>F4?1}^oy3WznD!|!Br)z>th7}N7v-GUjG7R>Mz%Ew z?2loC+`C9{I%I=7f9oPwkc}j4nj~ETRm7(V3gex6IzEcd?4J)U*r?7<#3drJ!P((r zZu89y?s7Qf?2nVL7caArhL#a@BqZE-&ZhMiRCZ=bmrw94bzYu>&NDRz-q5>gS2dSJ z_&+yjnISi*8k0t+ zu_Exorg#*sEu>PU>EnAZ=CJ}>mVNulgd^zYW;&E#3^7ei`#=#`p!w;Nr1%DpCcP9(q%mMYfD=ZO(h^$=z^>*#NEwbCPJv^ zVPqF+E5|jDcu1`;C1j$wg#`Qj9Vk?9lt7A3BbfuPQe+D+4PoaR4dbRXc&Sz0ZETG{4PlEE-y0lCqmkO?N?NZ`WDbE^st z=Ne)Lz^z@g`s#zlHFpH}j8+csHhd*bmiJcM++C4CfO5T>>ckkhq@v!QcYvMHkKPnEYE zHO81szZZg&vHvj&y3zB^`riNjA0;sB>x8uiK-&pPy4hBRP|_7`p{LxCL{SRJb?<>b z=p&GL=S?N3@Hv$G*s@qg0SjN>@cX&sWhdiy8gPq-=e+7Xhh(LjGLJWiHYlnQ)xNlc z@Q7q6Q2$7+lCbs9)|e%1{r@B<3{53R57;mAqFh1aHN&U77<$eW<_m`Q1Ca&^(0*+e zR%~z0K*Qbxu{*1YrOQ`_!0x-MjaUtV({|Z~Ld+5kA=*+$ism0mOPF*PZD8!B{RgTF zXxW7%aKGwJLOT7Wve2W_YF z998GO%17C`-0;cr?(3Ie{;bRo0uHBdb#n>THkU@$svkESDS0wdusC}ai#Lp2R1bUn zpv!^I4vEcZk{czT}Ht zagXNmjGBeczAw|1U!2cSUQD`J;1IEGLK`+_n5!hsdIWFP?E&ya=lwqC1f=Z#teg7! z zkaq|hW#!Q^aZX7(`tY9VxSPH;b$ZBCz1;Z{Z1zwzHFS4BkbvS*AEL19!3F4CJy9tSrR_ayw~y?bWn3%Q zFam5V`&sj0!edNb!!OBJLE|k1UECec?}h8RS_Rr@M<&AF&ufVg`VGSWOhsBuMX(-* zW|&z7seaPasaloH3?+lS7vkvYdo*D@(?G1<367xb^2u}|&K(E)LeEXo~*9XnWjUCG%nIk;56jV=&33PYJ zCZ^P}Qc=Ke<)7$Pls>%_ilZzl?dlW#aSqyB$-&amqq#)9ad#6O&h(rbSwI@tAnd9H zp6<@VuOkGOW}Ke|;;%OJN&r>msr@C%@d9(5rPI>qw7QgWp&VM8&$^n&Z3HmN9ZsF= zLx@Tw;C%K@5Ezm|^Xn3V^hSg2@)_Q5-);Y+so*qVlA%cL6wUQ_tz%G*q*u`#u3Lmu zn92#UKZ(0OHvuvofMUPt^Yatk(<5e)ag6gu8qme_GwEPifz73fi59Wo-OdCIffAp} zq-36y%6n9DfZygnbR;(jhvko7nal%YZk>>gHANfi=3cqdDv-B$l#P8KBf_3i8f#bG zu!5F^j{N$skDz{HYgk%yi@U=PqawA6xGud$gcHvnMSI1^Y{LeE^aM(PO-mnwD)=Q| zb!FK2mQ~#k0!ts1REqXvL2)MCmDeiU-@x10h(7MZ2dgIpUM?}ALjLIS{h&|Gt;89a z&u4M8Phk9LF*?r7WgU)9zY1R{GkZsm5JibF0l7p(NM+3Vw@z+LOr`>~O?VLKZ-iCb$?p8*^D>hNN6)MM z+a>mYUzZra1U8BpW0%1|zaxTn0xkeHN_pr+s{%T z0~AUpN6}WzN)`+q4XW^>c`V@^enLg{%<>;Z+71B?Jg!K=$q;&;6L!-6Wp3w~9}~W= zt~M~$Jv5I3N$AkEv-xw)r$g(2+$uncvIdbu${zxGk|<>cs{e|8{2bTgcUm{$YgDEI zV50Gr)q?iy?i8ixdc(|q23j`3Z322MO}^)OVqHSnV(9twu#&iZdn3-$ms=(fNAi0K175&VRGl=Uc=IGy!QYH&_DxOdU_{Qz7rwNG7=$Z^xB%OUkCbL zPXRPPk0OFbxk1j3?Rh0Ja7o1NF>UMKx?12DVg6g64Tg1NnOJH^IDdTZza}CyiwsaB zUUfX180qK+!$S5U#ZeEF8ewj=Jaf>1MJmV}`Mk>XZ%BIo6OxV^!Kmf4waXz-L%0lM z>5+4E0;)JZO^agR!dAlWP+WQw1_6D&$ejPY$ml=(-JH+zXP&dCn=(Z6H^O%DLb#G) zBEG>G;>OScj$?~cWgzr($V^6cz-WTq{4K7*8F4lKa7I{Mv!I>$(5HB%Wag|hPGrls zUS#&RY+{i;OX&wtDxgIbOqg=;)FgZE3jKa3Sq2!2R*H5Zt}z%=2_Ko=-~j@Ghm754 zFPLFyM$N-qpF9dz5T~|RpQ{Bzz>u?WS0n1d7-cLCt@ z%2o+N>g|xg;&dmbpJkX;lzb6}Qj`H4*R|<$;nOCZeQ>WtX5|YFS}YP5_14Ba!Mv8o z09pAvbFqB%`79$|jv~RpHW1pH2=cQGmjVy7PAe>6m@wp^_6NcA2~QU4zZ(5kLMRvrba_f_OfDEofaDW*^iG~! zAFWxoqq6RwvJv-zl8~o>%SL`fSLvIGEQF`^rxynSvAJ?)hq!345^rjGn~OixI+lJK zCfpOd*vW>XwZ)-ek}q0$MZZoR+h$klm!UDXvUQ+^>5v9>Q>4%57g5*##LF#Hz`0Ln zw|3400!o_#*=jd*u@rLIQEoXV*dY45CR)kE+l+u6i!M^?b0K`n^l9&1_1jrm>jm*( z1jPeocrT)ZxgpQoiZE8N9*#=^OimO+!5EO*4-Z|m$L$+B%a{<6j4T=r zJ=GzB%usX1ku5_VH4f788(KCA))#5_mTnTl$V>;WAx)p>nVpnY^4Bq0#yzC7j^ef*0M%Ia|FwM^L6 zz+#>MouPP)W{|@Y@0pu_37QMk5ZHdtHs`Eo1gJyNKFqba8+J01Mve*#e3O`sL=Ocq@#~JA2=3&(b{xfS?A_G@NPt zh37|Pm9dEA&jk=&*^r)BU&U1L7szfdxXVW1oHXa|_~M@sh;wniMQ>;69kTuq_k7fa zhG=SN?sJ?&z(sf-&j%DyN|Y>UMX8qDDx%05^jio3n*9yvXV!IX(=f|qVyerIP^d4*ruoLI3yaxzRre4d{sS5VcrQvEDF9bxS2m{W8< z0JBatn$>MdnEXI(QrId}Ta){W1s?on+s^TTDi7@mqzV~f>)imn2bV0(Zaej|B)Lga zZ1%@FLEU?nR#8vEl&nwl2u9w!1G%?3O=<^ge2mob zyxbVrW*DI1>@o$w^{R+7_DJL(RhVPSe+LAULUbx%^5tL z$VG=6p|-w?vvbq_MpUSP91j9yc0S(*PBwp7P`nhIeeQ+*A9ZB8p}Q=9ad+97O7hB_ zUxU|j^{AUGIj4mFp-`RieUT80wN@xOb?vBa3V3m43O`%Y`O^$O%g_ZA$gURv`g6*X zUZ-gd)cIKSXy7cg^`2%SmUom!b-RM2O(`!ZH4dQ~t)J$KL4m0!11Ju6tBkxv zE@;#=3{UBfx+4H+nQ0C>NhE5;-5EYhM5PEKp0)|>)3-$XEcI2KJvuoqO+LN`?iDb- zK^L6j)D}yA$MDx}9HCiPKp!dOTZ~iR51NAn13p!d{l4b1^fi}U1mk;8y_ED8%3to( zEw0zxC%YuOYJe>rANaw05SS4Y7Yf_+LZGF8vU z?7PxSn+t*qP|!GtIU{)7I2-sHP;b5U3>fV1c5n)P3wns`s5LyC3?g!HIuN>4saK=aFo$`i;KWJ-9r}U+7+Bmf$)#`vA=_oEfISP)T!XqF{0{pD-_Q zl`=4($gepju}|phhlwQ_GOV zh>-x=potCf84&qe65aC`6ZnLK4t1ZdZjPR#)qYgSj z;*U5KNy^hTQa<;}64%QAP#c=}la<1GDh5fktva78noiYwB7`$yv*AkB5dWMTxj+#M z3HyFWB-}};EF%e3=oN{2Xm~N=Iy*wK$qjtm?EGeha%A{)03A7-jGhrWm`ED|{#eK> zecJ>elYL~_%BjZU7#v?s10d5NxV7nW>#C-7NmTUz1&3eB4SVY<8&DZ01X+rcCr1vL z_dTaP%ecs#`ozJf&@?hSOFC!PZP2SHzJ=#vGAhx&%%c(@lJrwApNHsdo=KgDxt=2Z zwr}GK%U}QytSMI?2@ceLTXt@r+=Xd+TdLW!ap+X7WZ^Q4N%>F87aj&&tMdt@U}|wU zf}7;HJP#mf^v4kVdP*{}naF)G5Q>~^?o@>T`GZi|oYZ1c$hkL#G8s5xk6iCIY%VvJ zgXJ%+!7%c@Pf?(|?fGaV67I~_PZh*`mWH1{EMT0nH|_4uz^=-&<<#D=U7l~nzOfuE z^*A8iP|ZUL3y3R~sTr_1C_^bJwEc=1TO2-wzR^oyD&#BWVzO7%kgHD%Je%h|DLhM? zR+cX;L!(;GUjmAQ13={8Vw=JT;j{|6f+D7fQ(Xf{Nn?Q5N2cHBw6uMWN6;iHmZs(D zpaD>&V3i}9)k;Wk1L%G-!%!!EsVhdsh*nx9BYPsrKaG;9zx5>j)8^^jZ64AMW{iTX zV$Qb6f_h73ECV}?k;PH&_!!74mjb-|7N_hX+M^KI>N^1Q2Ao!VX8xCpP4g2bg5G2= zLXq4p|JT5pp)qo38}JE@$x9fDjZ#&LeYGE8sFwZQG@ zl;*xtPojO;cLk&|`B*pj^WUL*-lmu%&O=^Boj=~j>;BDE>VI;Tuv9uVEh`F#L7@oj zX52B1)r~?TENa7U znHh!TpfB$%AHFbR2xvkKRZ9c8Cc|XT?&m>GnCK_+E%f>1!4xaN&+9~HTAE9EJd+kg zUf^NmPNBEb8qBN_2*U#n)6k}#I*SOh_P$h;%5ryD%C8ZwiEQ$fi86jFQq2wuCIblo z2z&&@ed3Km;pLPw|;)=<3_ zf>2D#q6-hxIpKlCSE*sze8Xza7Jq?0&oXLvcoxm~_e`UJhH$o;kk%5n%I=cIieFj} zH2#S6_~ILMW<5~dVmCJ84)0Z3JNRd zmJ>8T2(_l#c@AarJ*;KvRv#_1Y!xs%KjpA=YZDefiOAe3siSscaL{yLh_QE~40eU$ zEK2Nj&uiRnx5AC3uW@_{>TsbJ1Uob|c~D=43QQQusjj$N!CMK@q-!9AHmH~jtdl(U zb4UqZ`o72Y$gsx(;+8MMT7Kqvf08|H(Cmz_;fV1TXUy zW3+xLM$m^eiNWK{AYAYx{YuJ)z>kmBZr8w2;1c7&LPuiqpw1@cn!|#9qb;vOW&J5r z;DdR(!krN-Er?OumADipfFvA|B}~@Jsmw@|{hVxAK_rVIQGdGUP#9c;NtVBqx#4K? z0^PdPSkv@*TjGS43pqtY*WsZ!(GBoGk(gw2fI&})QqB?obk?r*!QHMEcSxy-y|Xta z4(HfKS8g|@X^sW82#9UdH4`^jv;w>T0F-onkOQTENG4jk8%ivlk6cM9@`M`yjlzc>k1q2Dw zyIhrEYt&i9LD!Tl5C7OQbQq_V6TPr$J`2UPKZRo2p=uLhENQW#D2Gu`kXf=^%s z&)Zux@-L7FOv^lw6gsVFHX&NbbM(^gh-8*gC6WeS{V$@Ci?6ylfK<8r^ic`XO=DLn|%8C_i? ze+dveUtS^OV6C_I{9|Qh>5F_rHWiA8J;^zSQ)f{(u}qu<42P?=1T+oKPh5cGz&!hK zM%%{r(Z|~%bPoR;x{%D3#AxVD5^imUZvl_^v^a#m8c1M*!iFK6pEs;sl9PTyc@am<;z48u;6*D_^=TArSL7&?BT?#~ z=W{EcB&}wRL{`h#4vS4xq@;A`TaSq9v%)pQL9n7iYYb(p`XsYzXHyCW`X; z9PUqV^DJZLw4GU1IDs*a3)o~04QC=2DL6A>4q{GP;VoIb5WDkiXBvy2#cV@CO0K1C({<5R#c2}{ViFvBw+`%4}=PVqY!nG~2^qr{$U07iNz!M~{ zG#m;AwHQ@Jz1zqQOS~W)WZF4X4X*}1zJGg0e!}t}kp%#)tsR%M9EXc|b z=!MFIo;a1%Pz3Oehi+kY*U5;9mR}o1C)UEJz!zfeXZ#jE5ON-LZzY^7@k%?s%GMD2 zA{iOC$mDuni2Fr=AvT+DmpnOmx3I&YYI+||^Al#aX8)yaBAc5(Ng@gdnpA2|coykpxyZA!jJqTiv#tM2&~gu*G9w>Jrf^R#erKn*9t7l8wT|K_uOnRXyZ5O*y> zKl`(xgW}^eiQb0h$+E5ufiUUerjki%=mQX|a77109f6-vC)#qPh#TXaS)3=6FrR0& z1`OaffUE#0X5x$0yrFM2DpiQoyl!tZRjl}82wApO@l2g8!*4~I) zLAS2TebzW4amZUBLw6Y|5SALfqMOh~#(^+*ELB8b1$H$n$+lggzSr zOKTGF=sF_9p)4Fg(+xv;anW*$Ouc6KFsxtcX@Rhy`G0Oq?Zbnw%?NJnVRlQS zB~QSZCcs7`P$qqbQzg(!!wBiP67iq0pY$-;V?SqN2*(}S?X#b<>nB`pM@|Mk9a9V# zfs&^+lmMeB7UYuC048Sz6%@_^A;;Mm9vuQy zV_O2jivwZGGRQB0L}sa;Cm_?&#ONi)R9eNe!#T?*pPi-N2X5(lpoVOYm{{l^S?52n zUKUyi=f}CMRDX?Kt09uhZpAP**6f-|&{H>9%LfEiLpoVB>sRZ5OmehGr(8&8>MUi= zC58Hp=8-U}Zw>}(3O}%b9x$;HfjQ1@-JW;2^u5ESXtIKBux^d%PG*9NjB{E}NU4zZ z1}duYKl!zp$~A%jlGT%PhywQ0czV7svNhi))LF*s!2CV}wq;?*!HO2hPC1COBnRPP zi124hHa>cp9%_suE3`n;pz>3z1s>vRX|RmBOCjjq*4wkAAZy0-Q#@BZ=VrmW+R1T< zutPitxLBufG{YA_#$gq?DRBCJhuH{_(>W%%3WZ{fkG5jrpZ2su z?xc)n>U&Oa{r042NblR!k^E6(AFojg3H-DVB_XEq@(Q5{woPTRrslhz%KlRE5DAavWp@&RlnD# zw0|Rsz8gy`#$Q=7KqsNoNY5($Y)%_;qI6V=l!$I_k^hURR8b){jC>T=lQtAqv43C! zZ=c8fp>>nX1aPcN`vGDb)zK&vXy2--o7~R87CGdK9k^Z0qiF0zU9GkJd=bjIX5HrX zvy6>7gV(Z({yg%C_A|I0pS@`SM^*4|q_cx&hrzpbfv^Jr_b~cFj@Wma7izNm{!~)b ztmGXJLnVOTLAOAPV=kvgyd1k_1gs;X#`h>-ViYp=2f16Y1J^W&6v+^h|f)I%&+2RQV zj?8#Qq!a3T!VxX;SS9F^>4-umvrEk(3pb_hIicQ=Wo&x4IHOi)7?rYq?C>Re(fL`+ zAUNVj@l;GDsv4Zo$gHWlw_^x&w%n=ccw#$3lDBt#nyLFY(kolA2gJpcXDGH^*)t2W zgT8br{0oH=Ac4r{218ajL;`qB0u&}4ZvCh9dbZb;cX8S0@!@HEUd^adrYBgNAe_)k z_Qg&N519p*jyE$U=qqD9Oxua5r1g*~L#vjQllKl?Z?PClcUCGfJPvBhG#YVs(qq+M z$dH_Ub?(c z#QE@OSgV4?pRP0&ulpZRWE=dUbDMlK;EBw-Hb|Dw;F4-H?4l?I5_wp z9{x$sOJNDDos42XyXUlqFh}#Ny$g8m7&n9ozOjLYH7w|`JI`*2`eCZ8t^y6>5{(}v zcvu*M$N%ht=gc2;{iQo3;|=diqyLlM%CUWKkt`#xoN=$?FB1(j+Pg3cdyvOr5wUG@ zQHq?m$sip|cQ;WDrv~Y{bQ}?5KlwxL7PGQ6L34EnIf+ZiU6#AC@g92g==hY1iCW*p z1ZE(eUXhqkvx{Y!BS?(?oW#^G-=qHc9{=II_D%;C{`?CP@DO{<=2Ch(Cq-U{;GBPo zGXif#Pp+pK0i9lzJV32c6FcQ{-uolff;**&p z8*+l6nqeuGZE0`s&$>%kmi``29qe2IK7IBfN91_i$aIBIiHoECKoTv&qIpG$+@zMbfIwV-V;H-gEB%rfN>{H5IkEeD$d zbU1<(UcElR!he4WmOd?20$3C&hdBB?%hnZCLn%pW5?DOF4jW3Qz_YdauA^~T)s+&Q z;nVU_?%qZ(Km8cuQRVi)d4i;a_N6%XP%SlygJe32IgALQ&;w4r+q|zPSh~6cFxY)# zb#c9hKffEZ;gG^!TEx5gRyJP8Xv6;tW49Oi$oi~aS*AEX$ZeA#*sIVYu{f5%h((U zx~Nv{!y_Z`q{*r;Pa5pp3Jkml_56HQ2(24R^JxtKpreo9J8Pi!bgiE~^}bJ>We8j; z#MFb|1VM3+L#)@l_bLRStFO=;Ur!rAax=so)6{mNQ%JKf=aX6Q=y+7bcMJS1&Ec=n zTSJY60=S#U({tPHr_J9&Gl0S6% zG#d0U26K9K0-`XvTr&raP8XcX=8lXLg3-64WGN7>-i@Yn{(N#D`wGnjopbB$Wp|*n zUW)5iQq%zll%`&&w`0s6bk%edS9c_nh6W0ZQ}t=X;-S9%nP=(AAgmw{x46g*+52eR z^aw~G-f5${hUDSTeCh!KdCFMVZEDUaLV9U@j(5qjZu!bt26-vO!gO~D=I92fSnjh`51n4LCqp^}+zTH=imU}>zdJD2C6h9$NYvRW+(0R+W~ zyd86@w+5a~!?s>k_M1u1^UJ_ws(yQZ3VGggGM1K;k)O3~075kdmWxz7qbxJp9p!8y zt(Cy!Jn=~CvgJ0GHcaAW&hrRO^Svq1MEk$1-IQ9SiU2u4#=p6HI{6wg+now5{h