diff --git a/src/usortm/demux/streakout.py b/src/usortm/demux/streakout.py index 563f405..c3d1304 100644 --- a/src/usortm/demux/streakout.py +++ b/src/usortm/demux/streakout.py @@ -46,6 +46,25 @@ logger = logging.getLogger(__name__) +# --------------------------------------------------------------------------- +# Pileup row encoding +# --------------------------------------------------------------------------- +# A pileup row holds one (symbol, is_match) tuple per reference position. +# Deletions and uncovered positions must stay distinct: a deletion is positive +# evidence that the read disagrees with the reference, while an uncovered +# position is merely absent data. Collapsing the two makes a read with a large +# deletion score 100% identical to the reference. +PILEUP_NOCOV = "-" # read does not span this reference position +PILEUP_DEL = "*" # position deleted within the read's aligned span + +# Minimum read depth before a column may be flagged as a problem position. +# Without this, a single read covering a single position flags it at 100%. +PILEUP_MIN_FLAG_DEPTH = 3 + +# Fraction of covering reads that must disagree with the reference to flag. +PILEUP_MISMATCH_THRESHOLD = 0.10 + + # --------------------------------------------------------------------------- # CIGAR classification (mirrors utils.extract_matches logic) # --------------------------------------------------------------------------- @@ -818,45 +837,13 @@ def _build_pileup_from_bam( """ if min_overlap_pos < 0: min_overlap_pos = ref_len // 2 - rows = [] - try: - with pysam.AlignmentFile(bam_path, "rb", check_sq=False) as bf: - for read in bf.fetch(until_eof=True): - if read.query_name not in read_names: - continue - if read.is_unmapped or read.is_secondary or read.is_supplementary: - continue - if min_overlap_pos and ( - read.reference_end is None - or read.reference_start is None - or read.reference_end <= min_overlap_pos - or read.reference_start >= min_overlap_pos - ): - continue - row = [("-", True)] * ref_len - # Use get_aligned_pairs() without with_seq=True to avoid - # requiring the MD tag, then look up the reference base from - # ref_seq directly. - pairs = read.get_aligned_pairs() - for qpos, rpos in pairs: - if rpos is None or rpos >= ref_len: - continue - if qpos is None: - row[rpos] = ("-", True) - else: - qbase = read.query_sequence[qpos] - is_match = qbase.upper() == ref_seq[rpos].upper() - row[rpos] = (qbase, is_match) - rows.append(row) - except Exception as exc: - logger.warning("BAM pileup extraction failed: %s", exc) - - # Cluster reads by mismatch pattern (see _build_pileup_grid). - if rows: - rows.sort(key=lambda row: "".join( - "." if m or b == "-" else b.upper() for b, m in row - )) - + # with_seq=False avoids requiring an MD tag; the reference base is looked + # up from ref_seq instead. + rows = _rows_from_aligned_bam( + bam_path, ref_seq, ref_len, min_overlap_pos, + read_names=read_names, with_seq=False, check_sq=False, + ) + _sort_rows_by_mismatch(rows) return rows @@ -927,7 +914,9 @@ def _make_section(ginfo: dict, grp: pd.DataFrame) -> Optional[dict]: ) return { "ref_id": ref_id, - "n_reads": len(pileup_rows), + # n_reads is the group size; n_aligned is how many produced a row. + "n_reads": len(grp), + "n_aligned": len(pileup_rows), "frac": frac, "status": status, "is_recoverable": is_recoverable, @@ -951,19 +940,26 @@ def _make_section(ginfo: dict, grp: pd.DataFrame) -> Optional[dict]: status = "Clean" if _cigar_is_clean(ginfo.get("cigar")) else "Mutation" variant_fasta = os.path.join(single_ref_dir, f"{ginfo['variant']}.fasta") - if os.path.exists(variant_fasta) and minimap2_path and samtools_path: - pileup_rows = _build_pileup_from_bam_realign( - bam_path, read_names, variant_fasta, - minimap2_path, samtools_path, + if not (os.path.exists(variant_fasta) and minimap2_path and samtools_path): + # No reference to display against: skip the group, matching + # _make_section rather than emitting an empty section. + logger.warning( + "Pileup: skipping group %s (missing FASTA or aligner)", + ginfo.get("variant"), ) - ref_seq = str(next(SeqIO.parse(variant_fasta, "fasta")).seq) - else: - pileup_rows = [] - ref_seq = "" + continue + + pileup_rows = _build_pileup_from_bam_realign( + bam_path, read_names, variant_fasta, + minimap2_path, samtools_path, + ) + ref_seq = str(next(SeqIO.parse(variant_fasta, "fasta")).seq) group_sections.append({ "ref_id": ginfo["variant"], - "n_reads": len(pileup_rows), + # n_reads is the group size; n_aligned is how many produced a row. + "n_reads": len(read_names), + "n_aligned": len(pileup_rows), "frac": frac, "status": status, "is_recoverable": is_recoverable, @@ -1002,6 +998,157 @@ def _make_section(ginfo: dict, grp: pd.DataFrame) -> Optional[dict]: f.write(html) +def _run_pileup_alignment( + fq_path: str, + ref_target: str, + out_bam: str, + minimap2_path: str, + samtools_path: str, +) -> bool: + """Align *fq_path* to *ref_target*, writing a sorted, indexed BAM. + + Returns True on success. The parent's copy of minimap2's stdout is closed + once samtools owns it, so minimap2 gets EPIPE if samtools exits early + instead of leaving ``wait()`` blocked forever. Both exit codes are + checked: a silently failed pipeline used to look like an empty pileup. + """ + try: + mm2 = subprocess.Popen( + [minimap2_path, "-a", "--MD", "--secondary=no", ref_target, fq_path], + stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, + ) + except Exception as exc: + logger.warning("Pileup alignment could not start minimap2: %s", exc) + return False + + try: + sort_proc = subprocess.run( + [samtools_path, "sort", "-o", out_bam], + stdin=mm2.stdout, stderr=subprocess.DEVNULL, check=False, + ) + except Exception as exc: + logger.warning("Pileup alignment failed during samtools sort: %s", exc) + mm2.kill() + mm2.wait() + return False + finally: + if mm2.stdout is not None: + mm2.stdout.close() + + mm2_rc = mm2.wait() + if mm2_rc != 0: + logger.warning("minimap2 exited %s during pileup alignment", mm2_rc) + return False + if sort_proc.returncode != 0: + logger.warning( + "samtools sort exited %s during pileup alignment", sort_proc.returncode + ) + return False + + index_proc = subprocess.run( + [samtools_path, "index", out_bam], + stderr=subprocess.DEVNULL, check=False, + ) + if index_proc.returncode != 0: + # Only sequential reads follow, which do not need the index. + logger.debug("samtools index exited %s; continuing", index_proc.returncode) + return True + + +def _rows_from_aligned_bam( + bam_path: str, + ref_seq: str, + ref_len: int, + min_overlap_pos: int, + read_names: set = None, + with_seq: bool = True, + check_sq: bool = True, +) -> list: + """Parse an aligned BAM into pileup rows, one row per aligned read. + + Each row is a list of ``(symbol, is_match)`` tuples indexed by reference + position. *symbol* is a base character, :data:`PILEUP_DEL` for a deletion + inside the read's aligned span, or :data:`PILEUP_NOCOV` where the read does + not reach. + + Insertions relative to the reference cannot be represented in this + column-per-reference-position layout and are dropped. The rendered page + says so, so that a clean-looking pileup is not misread as "no insertions". + + This is the single place the symbol encoding is applied; every pileup + builder routes through it so the encoding cannot drift between them. + + Args: + read_names: When given, only reads with these names are included, and + secondary/supplementary alignments are skipped. + with_seq: Pass ``with_seq=True`` to ``get_aligned_pairs``, which needs + an MD tag. Set False to look the reference base up from *ref_seq* + instead, for BAMs written without MD. + check_sq: Passed to :class:`pysam.AlignmentFile`; False tolerates a + missing ``@SQ`` header. + """ + rows = [] + try: + with pysam.AlignmentFile(bam_path, "rb", check_sq=check_sq) as bf: + # until_eof=True does not require an index, so a failed + # `samtools index` still yields a readable pileup. + for read in bf.fetch(until_eof=True): + if read_names is not None: + if read.query_name not in read_names: + continue + if read.is_secondary or read.is_supplementary: + continue + if read.is_unmapped: + continue + # Skip reads that don't span the midpoint of the reference. + # 5' concatemers end before the midpoint; 3' concatemers + # start after it. Only full-length reads cross it from + # both sides and cover the variable region. + if min_overlap_pos and ( + read.reference_end is None + or read.reference_start is None + or read.reference_end <= min_overlap_pos + or read.reference_start >= min_overlap_pos + ): + continue + row = [(PILEUP_NOCOV, False)] * ref_len + pairs = ( + read.get_aligned_pairs(with_seq=True) if with_seq + else read.get_aligned_pairs() + ) + for pair in pairs: + qpos, rpos = pair[0], pair[1] + if rpos is None or rpos >= ref_len: + continue # insertion, or beyond the reference end + if qpos is None: + row[rpos] = (PILEUP_DEL, False) + else: + qbase = read.query_sequence[qpos] + is_match = qbase.upper() == ref_seq[rpos].upper() + row[rpos] = (qbase, is_match) + rows.append(row) + except Exception as exc: + logger.warning("BAM parsing failed: %s", exc) + + return rows + + +def _sort_rows_by_mismatch(rows: list) -> None: + """Sort *rows* in place so reads sharing a mismatch pattern are adjacent. + + Key: "." for a match or for no coverage, the base for a mismatch, ``*`` for + a deletion. Identical patterns sort together and dots sort before letters, + so the cleanest reads come first and subpopulations stack visibly. + """ + def _mismatch_key(row): + return "".join( + "." if (is_match or sym == PILEUP_NOCOV) else sym.upper() + for sym, is_match in row + ) + + rows.sort(key=_mismatch_key) + + def _build_pileup_from_bam_realign( bam_path: str, read_names: set, @@ -1055,59 +1202,15 @@ def _build_pileup_from_bam_realign( return [] # Align to target variant reference - try: - mm2 = subprocess.Popen( - [minimap2_path, "-a", "--MD", "--secondary=no", target_fasta, fq_path], - stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, - ) - subprocess.run( - [samtools_path, "sort", "-o", out_bam], - stdin=mm2.stdout, stderr=subprocess.DEVNULL, check=False, - ) - mm2.wait() - subprocess.run( - [samtools_path, "index", out_bam], - stderr=subprocess.DEVNULL, check=False, - ) - except Exception as exc: - logger.warning("Re-alignment for pileup failed: %s", exc) + if not _run_pileup_alignment( + fq_path, target_fasta, out_bam, minimap2_path, samtools_path + ): return [] - # Parse aligned BAM into pileup rows - rows = [] - try: - with pysam.AlignmentFile(out_bam, "rb") as bf: - for read in bf: - if read.is_unmapped: - continue - if min_overlap_pos and ( - read.reference_end is None - or read.reference_start is None - or read.reference_end <= min_overlap_pos - or read.reference_start >= min_overlap_pos - ): - continue - row = [("-", True)] * ref_len - pairs = read.get_aligned_pairs(with_seq=True) - for qpos, rpos, rbase in pairs: - if rpos is None or rpos >= ref_len: - continue - if qpos is None: - row[rpos] = ("-", True) - else: - qbase = read.query_sequence[qpos] - is_match = qbase.upper() == ref_seq[rpos].upper() - row[rpos] = (qbase, is_match) - rows.append(row) - except Exception as exc: - logger.warning("Re-aligned BAM pileup parsing failed: %s", exc) - - # Cluster reads by mismatch pattern (see _build_pileup_grid). - if rows: - rows.sort(key=lambda row: "".join( - "." if m or b == "-" else b.upper() for b, m in row - )) + rows = _rows_from_aligned_bam(out_bam, ref_seq, ref_len, min_overlap_pos) + # Cluster reads by mismatch pattern so subpopulations are adjacent. + _sort_rows_by_mismatch(rows) return rows @@ -1129,7 +1232,8 @@ def _build_pileup_grid( (typically >1 kb). Pass 0 to disable the filter entirely. Returns a list of rows, where each row is a list of - (base_char, is_match) tuples indexed by reference position. + (symbol, is_match) tuples indexed by reference position. See + :func:`_rows_from_aligned_bam` for the symbol encoding. """ if min_overlap_pos < 0: min_overlap_pos = ref_len // 2 @@ -1144,56 +1248,12 @@ def _build_pileup_grid( # Align (use pre-built .mmi index if available) mm2_ref = ref_index if ref_index else ref_fasta - try: - mm2 = subprocess.Popen( - [minimap2_path, "-a", "--MD", "--secondary=no", mm2_ref, fq_path], - stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, - ) - subprocess.run( - [samtools_path, "sort", "-o", bam_path], - stdin=mm2.stdout, stderr=subprocess.DEVNULL, check=False, - ) - mm2.wait() - subprocess.run( - [samtools_path, "index", bam_path], - stderr=subprocess.DEVNULL, check=False, - ) - except Exception as exc: - logger.warning("Pileup alignment failed: %s", exc) + if not _run_pileup_alignment( + fq_path, mm2_ref, bam_path, minimap2_path, samtools_path + ): return [] - # Parse BAM for pileup - rows = [] - try: - with pysam.AlignmentFile(bam_path, "rb") as bf: - for read in bf: - if read.is_unmapped: - continue - # Skip reads that don't span the midpoint of the reference. - # 5' concatemers end before the midpoint; 3' concatemers - # start after it. Only full-length reads cross it from - # both sides and cover the variable region. - if min_overlap_pos and ( - read.reference_end is None - or read.reference_start is None - or read.reference_end <= min_overlap_pos - or read.reference_start >= min_overlap_pos - ): - continue - row = [("-", True)] * ref_len # default: gap - pairs = read.get_aligned_pairs(with_seq=True) - for qpos, rpos, rbase in pairs: - if rpos is None or rpos >= ref_len: - continue - if qpos is None: - row[rpos] = ("-", True) # deletion - else: - qbase = read.query_sequence[qpos] - is_match = qbase.upper() == ref_seq[rpos].upper() - row[rpos] = (qbase, is_match) - rows.append(row) - except Exception as exc: - logger.warning("BAM parsing failed: %s", exc) + rows = _rows_from_aligned_bam(bam_path, ref_seq, ref_len, min_overlap_pos) if not rows: logger.warning( @@ -1202,16 +1262,7 @@ def _build_pileup_grid( ) # Cluster reads by mismatch pattern so subpopulations are adjacent. - # Key: "." for match/gap, actual base for mismatch → identical patterns - # sort together. Fewest mismatches first (dots sort before letters). - if rows: - def _mismatch_key(row): - return "".join( - "." if is_match or base == "-" else base.upper() - for base, is_match in row - ) - rows.sort(key=_mismatch_key) - + _sort_rows_by_mismatch(rows) return rows @@ -1221,11 +1272,23 @@ def _render_pileup_html(well_pos: str, candidate: dict, """Render the pileup HTML page for one well. Uses an HTML5 canvas matrix: each read is a row of colored cells. - Green = match, per-base color = mismatch, light gray = gap. + Match = neutral gray, per-base color = mismatch, dark red = deletion, + white = position not covered by the read. + + Insertions relative to the reference are not shown (the matrix has one + column per reference position); the page states this in its legend. """ import html as _html import json as _json + def _js(obj) -> str: + """JSON for embedding in an inline `` would otherwise close the block early. + """ + return _json.dumps(obj).replace("", "<\\/") + flanks_js = "null" if flank_lengths and (flank_lengths[0] or flank_lengths[1]): flanks_js = f"[{flank_lengths[0]},{flank_lengths[1]}]" @@ -1244,113 +1307,151 @@ def _render_pileup_html(well_pos: str, candidate: dict, else: status_class = "status-other" - # Compute per-read identity from pileup data + ref_seq = g["ref_seq"] + ref_len = len(ref_seq) + pileup_rows = g["pileup_rows"] + + # Read counts. The size of the group and the number of reads that + # produced an aligned row are different numbers: the midpoint filter in + # the row builder drops concatemers. Reporting only the latter beside a + # fraction derived from the former makes the two contradict each other. + n_aligned = g.get("n_aligned", len(pileup_rows)) + n_group = g.get("n_reads", n_aligned) + + # Identity pooled over reads. Positions a read does not cover are + # excluded, but deletions inside its aligned span count against it — + # they are real disagreements with the reference, not missing data. identity_str = "" - if g["pileup_rows"]: - total_bases = 0 - total_matches = 0 - for row in g["pileup_rows"]: - aligned = [(b, m) for b, m in row if b != "-"] - total_bases += len(aligned) - total_matches += sum(1 for _, m in aligned if m) - if total_bases > 0: - identity = total_matches / total_bases - identity_str = f" · Read identity: {identity:.1%}" - - ref_len = len(g["ref_seq"]) - - header = ( - f'