aggregate.py

#!/usr/bin/env python3
"""
Aggregates raw JSON output from all fetcher scripts into a single compact
context file for LLM consumption. Optionally applies AI enrichment passes
(analyze_diff, extract_thread, link_tickets, classify_review) when their
output files are passed via --enrichments.

Responsibilities:
  - PR categorization:
      Created PR         = I opened it
      Code Review        = I reviewed someone else's PR
      Updated/Refined PR = I commented on my own PR (inferred from commits on owned PRs)
  - Gap detection:     Find weekdays in range with no recorded activity
  - Timeline hints:    Annotate gaps near Linear ticket creation → likely local dev
  - Enrichment:        Apply AI sub-agent results to replace/enhance generic event labels
  - Output:            Compact plaintext, minimizing tokens sent to the LLM
"""

import json
import argparse
import sys
from datetime import datetime, timedelta
from collections import defaultdict


def load_json(path):
    try:
        with open(path) as f:
            return json.load(f)
    except Exception as e:
        print(f"[aggregate] Could not load {path}: {e}", file=sys.stderr)
        return {}


# ─── GitHub ───────────────────────────────────────────────────────────────────

def process_github(data, handle):
    events = []

    # PRs the user opened
    for pr in data.get("prs_opened", []):
        date = (pr.get("createdAt") or "")[:10]
        repo = pr.get("repository", {}).get("fullName", pr.get("repository", {}).get("nameWithOwner", "?"))
        draft  = " [DRAFT]"  if pr.get("isDraft")  else ""
        merged = " [MERGED]" if pr.get("mergedAt")  else ""
        events.append({
            "date":    date,
            "service": "GitHub",
            "type":    "Created PR",
            "detail":  f"[{repo}]{draft}{merged} {pr.get('title', '?')} — {pr.get('url', '')}",
        })

    # PRs the user reviewed (on other people's PRs)
    for pr in data.get("pr_reviews", []):
        date   = (pr.get("updatedAt") or pr.get("createdAt") or "")[:10]
        repo   = pr.get("repository", {}).get("fullName", pr.get("repository", {}).get("nameWithOwner", "?"))
        author = pr.get("author", {}).get("login", "?")
        events.append({
            "date":    date,
            "service": "GitHub",
            "type":    "Code Review",
            "detail":  f"[{repo}] Reviewed @{author}'s PR: {pr.get('title', '?')} — {pr.get('url', '')}",
            "_pr_url": pr.get("url", ""),  # used by classify_review enrichment
        })

    # Commits — group by (date, repo) to keep token count low
    by_date_repo = defaultdict(list)
    for c in data.get("commits", []):
        key = (c.get("date", "?"), c.get("repo", "?"))
        by_date_repo[key].append(c.get("message", ""))

    for (date, repo), msgs in sorted(by_date_repo.items()):
        sample = msgs[0][:80] if msgs else ""
        more   = f" (+{len(msgs)-1} more)" if len(msgs) > 1 else ""
        events.append({
            "date":    date,
            "service": "GitHub",
            "type":    "Commits",
            "detail":  f"[{repo}] {len(msgs)} commit(s){more}. e.g. \"{sample}\"",
            "_repo":   repo,  # used by analyze_diff enrichment
        })

    return events


# ─── Linear ───────────────────────────────────────────────────────────────────

def process_linear(data):
    events = []
    linear_tickets = []  # for gap-detection cross-reference

    for issue in data.get("issues_assigned", []) + data.get("issues_created_only", []):
        updated    = (issue.get("updatedAt") or "")[:10]
        created    = (issue.get("createdAt") or "")[:10]
        state      = issue.get("state", {}).get("name", "?")
        state_type = issue.get("state", {}).get("type", "")
        project    = (issue.get("project") or {}).get("name", "")
        team       = (issue.get("team")    or {}).get("name", "")
        labels     = ", ".join(l["name"] for l in (issue.get("labels") or {}).get("nodes", []))
        area       = project or team or "?"
        label_str  = f" [{labels}]" if labels else ""

        events.append({
            "date":    updated,
            "service": "Linear",
            "type":    f"Issue — {state}",
            "detail":  f"[{area}]{label_str} {issue.get('title', '?')} — {issue.get('url', '')}",
        })

        # Store for gap cross-reference (identifier = ENG-XXX style id)
        linear_tickets.append({
            "id":         issue.get("identifier", issue.get("id", "")),
            "title":      issue.get("title", "?"),
            "created":    created,
            "updated":    updated,
            "state_type": state_type,
        })

    for comment in data.get("comments", []):
        date  = (comment.get("createdAt") or "")[:10]
        issue = comment.get("issue") or {}
        events.append({
            "date":    date,
            "service": "Linear",
            "type":    "Issue Comment",
            "detail":  f"Commented on: {issue.get('title', '?')} — {issue.get('url', '')}",
        })

    return events, linear_tickets


# ─── Notion ───────────────────────────────────────────────────────────────────

def process_notion(data):
    events = []
    for page in data.get("pages", []):
        date   = page.get("last_edited_time") or page.get("created_time") or ""
        action = "Created" if page.get("created_by_me") and page.get("created_time") == page.get("last_edited_time") else "Edited"
        if page.get("created_by_me") and not page.get("edited_by_me"):
            action = "Created"
        events.append({
            "date":    date,
            "service": "Notion",
            "type":    f"Page {action}",
            "detail":  f"{page.get('title', 'Untitled')} — {page.get('url', '')}",
        })
    return events


# ─── Slack ────────────────────────────────────────────────────────────────────

def process_slack(data):
    events = []
    # Group by (date, channel) to keep token count low
    by_date_chan = defaultdict(list)
    for msg in data.get("messages", []):
        key = (msg.get("date", "?"), msg.get("channel", "?"))
        by_date_chan[key].append(msg.get("text", ""))

    for (date, channel), texts in sorted(by_date_chan.items()):
        sample    = texts[0][:80].replace("\n", " ") if texts else ""
        is_thread = any(
            m.get("thread_ts")
            for m in data.get("messages", [])
            if m.get("channel") == channel and m.get("date") == date
        )
        ctx = " [thread replies included]" if is_thread else ""
        events.append({
            "date":    date,
            "service": "Slack",
            "type":    "Messages Sent",
            "detail":  f"[#{channel}]{ctx} {len(texts)} msg(s). e.g. \"{sample}\"",
        })
    return events


# ─── Enrichment application ───────────────────────────────────────────────────

def apply_diff_enrichments(events, diff_results):
    """Replace generic commit summaries with Claude's semantic analysis."""
    lookup = {}
    for r in diff_results:
        key = (r.get("date", ""), r.get("repo", ""))
        if key[0] and key[1]:
            lookup[key] = r

    for event in events:
        if event.get("service") != "GitHub" or event.get("type") != "Commits":
            continue
        key = (event.get("date", ""), event.get("_repo", ""))
        if key not in lookup:
            continue
        r    = lookup[key]
        repo = event["_repo"]
        # Preserve the commit count from the existing detail line
        detail = event["detail"]
        count_str = detail.split("commit(s)")[0].split("] ")[-1].strip() + " commit(s)"
        change_type = r.get("change_type", "unknown")
        summary     = r.get("semantic_summary", "")
        event["detail"] = f"[{repo}] {count_str} — {summary} [{change_type}]"
        event["type"]   = f"Commits [{change_type}]"


def apply_review_enrichments(events, review_results):
    """Replace blank 'Code Review' label with depth and 1-sentence summary."""
    lookup = {r.get("pr_url", ""): r for r in review_results if r.get("pr_url")}

    for event in events:
        if event.get("service") != "GitHub" or event.get("type") != "Code Review":
            continue
        url = event.get("_pr_url", "")
        if not url or url not in lookup:
            continue
        r       = lookup[url]
        depth   = r.get("depth", "?")
        summary = r.get("summary", "")
        event["type"] = f"Code Review [{depth}]"
        if summary:
            event["detail"] = event["detail"] + f" | {summary}"


def build_thread_events(thread_results):
    """Convert non-noise thread classification results into events."""
    events = []
    for r in thread_results:
        if r.get("thread_type") == "noise":
            continue
        channel     = r.get("channel", "?")
        date        = r.get("date", "?")
        thread_type = r.get("thread_type", "discussion")
        summary     = r.get("summary") or ""
        action      = r.get("action_item_for_me")

        detail = f"[#{channel}] {thread_type.title()}: {summary}"
        if action:
            detail += f" → Action: {action}"

        events.append({
            "date":    date,
            "service": "Slack",
            "type":    f"Thread [{thread_type}]",
            "detail":  detail,
        })
    return events


# ─── Gap detection ────────────────────────────────────────────────────────────

def detect_gaps_with_hints(events, linear_tickets, since, until, confirmed_links=None):
    """
    Find weekdays with no activity and annotate with likely work deductions
    based on nearby Linear tickets and (optionally) confirmed ticket-PR links.
    """
    active_dates = set(e["date"] for e in events if e.get("date"))

    start = datetime.strptime(since, "%Y-%m-%d")
    end   = datetime.strptime(until, "%Y-%m-%d")

    # Build a lookup from ticket_id → confirmed PR link (for richer gap hints)
    link_by_ticket = {}
    if confirmed_links:
        for link in confirmed_links:
            if link.get("linked"):
                tid = link.get("ticket_id", "")
                if tid:
                    link_by_ticket[tid] = link

    gap_hints = []
    current   = start

    while current <= end:
        date_str   = current.strftime("%Y-%m-%d")
        is_weekday = current.weekday() < 5  # Mon–Fri

        if is_weekday and date_str not in active_dates:
            hint = "No recorded activity."

            for ticket in linear_tickets:
                ticket_created = ticket.get("created", "")
                ticket_updated = ticket.get("updated", "")
                # Gap day is between ticket creation and its next recorded event
                if ticket_created < date_str <= ticket_updated:
                    ticket_id = ticket.get("id", "")
                    if ticket_id and ticket_id in link_by_ticket:
                        link  = link_by_ticket[ticket_id]
                        conf  = link.get("confidence", "?")
                        pr_url = link.get("pr_url", "?")
                        hint = (
                            f"Local development for Linear ticket \"{ticket['title']}\" — "
                            f"confirmed ({conf} confidence) to implement {pr_url}."
                        )
                    else:
                        hint = (
                            f"Likely local development/implementation for Linear ticket: "
                            f"\"{ticket['title']}\" (created {ticket_created}, "
                            f"state updated {ticket_updated})."
                        )
                    break

            gap_hints.append({"date": date_str, "hint": hint})

        current += timedelta(days=1)

    return gap_hints


# ─── Formatting ───────────────────────────────────────────────────────────────

def format_context(events, gap_hints, since, until):
    lines = [
        "WORK SUMMARY CONTEXT",
        f"Period : {since} → {until}",
        f"Built  : {datetime.now().strftime('%Y-%m-%d %H:%M')}",
        "",
    ]

    by_date = defaultdict(list)
    for e in events:
        if e.get("date"):
            by_date[e["date"]].append(e)

    if by_date:
        lines.append("=== ACTIVITY LOG ===")
        for date in sorted(by_date.keys()):
            lines.append(f"\n[{date}]")
            for e in by_date[date]:
                lines.append(f"  {e['service']:8s} | {e['type']:30s} | {e['detail']}")
    else:
        lines.append("No activity found in this period.")

    if gap_hints:
        lines.append("\n=== WEEKDAY GAPS (no recorded activity) ===")
        for g in gap_hints:
            lines.append(f"  {g['date']} — {g['hint']}")

    lines.append("")
    return "\n".join(lines)


# ─── Main ─────────────────────────────────────────────────────────────────────

def main():
    parser = argparse.ArgumentParser(description="Aggregate fetcher outputs into LLM context")
    parser.add_argument("--github")
    parser.add_argument("--linear")
    parser.add_argument("--notion")
    parser.add_argument("--slack")
    parser.add_argument("--enrichments", action="append", default=[],
                        metavar="FILE",
                        help="Enrichment JSON file (can be specified multiple times)")
    parser.add_argument("--since",  required=True)
    parser.add_argument("--until",  required=True)
    parser.add_argument("--handle", required=True, help="GitHub handle (for PR categorization)")
    parser.add_argument("--output", default="context.txt")
    args = parser.parse_args()

    all_events     = []
    linear_tickets = []

    if args.github:
        data = load_json(args.github)
        if data.get("service") == "github":
            all_events.extend(process_github(data, args.handle))
        else:
            print(f"[aggregate] GitHub data missing or malformed in {args.github}", file=sys.stderr)

    if args.linear:
        data = load_json(args.linear)
        if data.get("service") == "linear":
            evts, tickets = process_linear(data)
            all_events.extend(evts)
            linear_tickets.extend(tickets)
        else:
            print(f"[aggregate] Linear data missing or malformed in {args.linear}", file=sys.stderr)

    if args.notion:
        data = load_json(args.notion)
        if data.get("service") == "notion":
            all_events.extend(process_notion(data))
        else:
            print(f"[aggregate] Notion data missing or malformed in {args.notion}", file=sys.stderr)

    if args.slack:
        data = load_json(args.slack)
        if data.get("service") == "slack":
            all_events.extend(process_slack(data))
        else:
            print(f"[aggregate] Slack data missing or malformed in {args.slack}", file=sys.stderr)

    # ── Load and apply enrichments ────────────────────────────────────────────

    enrichments = {}  # enrichment_name -> list of result objects
    for epath in (args.enrichments or []):
        try:
            data  = load_json(epath)
            etype = data.get("enrichment")
            if etype:
                enrichments[etype] = data.get("results", [])
                print(
                    f"[aggregate] Loaded enrichment '{etype}' "
                    f"({len(enrichments[etype])} result(s)) from {epath}",
                    file=sys.stderr
                )
        except Exception as e:
            print(f"[aggregate] Could not load enrichment {epath}: {e}", file=sys.stderr)

    if "analyze_diff" in enrichments:
        apply_diff_enrichments(all_events, enrichments["analyze_diff"])

    if "classify_review" in enrichments:
        apply_review_enrichments(all_events, enrichments["classify_review"])

    if "extract_thread" in enrichments:
        thread_events = build_thread_events(enrichments["extract_thread"])
        all_events.extend(thread_events)

    confirmed_links = enrichments.get("link_tickets", [])
    gap_hints = detect_gaps_with_hints(all_events, linear_tickets, args.since, args.until, confirmed_links)
    context   = format_context(all_events, gap_hints, args.since, args.until)

    with open(args.output, "w") as f:
        f.write(context)

    event_count = len(all_events)
    gap_count   = len(gap_hints)
    print(f"[aggregate] {event_count} events, {gap_count} gap day(s) → {args.output}", file=sys.stderr)
    # Print the output path to stdout for the orchestrator to capture
    print(args.output)


if __name__ == "__main__":
    main()