feat: add two-step skills-graph job matching experiment

backend/experiments/two_step_reterival_skills_graphs/.gitignore

@@ -0,0 +1,5 @@
+__pycache__/
+*.pyc
+dashboard/output/*
+!dashboard/output/.gitkeep
+taxonomy_graph_viz.html

backend/experiments/two_step_reterival_skills_graphs/README.md

@@ -0,0 +1,103 @@
+# Two-step skills-graph job matching
+
+Standalone experiment: match Njila users to jobs using a **skill taxonomy graph** with a sequential two-step ranker.
+
+```
+exact label overlap  →  weighted graph distance (Dijkstra)
+     (precision)              (recall on the rest)
+```
+
+| Ranker | Module | Role |
+|--------|--------|------|
+| **Final** (production) | `final.py` | Exact matches first, then up to 15 graph recommendations from jobs not already retrieved |
+| **Exact match** (baseline) | `exact_match.py` | Label overlap only: ≥2 skills and ≥10% job skill coverage |
+| **Graph Dijkstra** (baseline) | `graph_dijkstra.py` | Weighted shortest path per job skill from any mapped user skill |
+
+## Data
+
+| File | Description |
+|------|-------------|
+| `data/njila_users.jsonl` | Njila user profiles (one JSON object per line) |
+| `data/ranked_jobs_v2.json` | Job pool with mapped skills |
+| `backend/resources/skill_taxonomy/` | Shared repo taxonomy (CSV hierarchy) |
+
+## Layout
+
+```
+two_step_reterival_skills_graphs/
+├── README.md
+├── requirements.txt
+├── paths.py                # Shared data paths
+├── main.py                 # CLI: final ranker for one user
+├── final.py                # Production ranker
+├── exact_match.py          # Step 1 baseline
+├── graph_dijkstra.py       # Step 2 baseline
+├── registry.py             # run_final(), run_all(), dashboard helpers
+├── graph_engine/
+├── data/
+└── dashboard/
+    ├── build_dashboard.py
+    ├── run_njila_dashboard.py
+    └── output/             # Generated (gitignored)
+```
+
+### `graph_engine/`
+
+| File | Purpose |
+|------|---------|
+| `build_graph.py` | Taxonomy CSV → weighted NetworkX graph |
+| `context.py` | `load_context()`, map user/job skills to nodes |
+| `job_loader.py` | `Job` types, load `ranked_jobs_v2.json` |
+| `user_profile.py` | Parse Njila users from JSONL |
+| `rec_format.py` | Recommendation dict shape for rankers + dashboard |
+| `models.py` | Taxonomy node/edge enums and CSV row types |
+
+## Setup
+
+```bash
+pip install -r requirements.txt
+```
+
+## Commands
+
+### One user (production ranker)
+
+```bash
+python3 main.py <user_id>
+```
+
+### Full Njila batch + comparison dashboard
+
+```bash
+python3 dashboard/run_njila_dashboard.py
+```
+
+Writes `dashboard/output/final_dashboard.json` and `.html` (regenerate locally; not committed).
+
+Open the HTML in a browser. Default columns: **Exact match** vs **Final**. Use the dropdowns to compare any method.
+
+### Regenerate HTML only
+
+```bash
+python3 -c "
+from pathlib import Path
+from dashboard.build_dashboard import write_dashboard
+write_dashboard(
+    Path('dashboard/output/final_dashboard.json'),
+    Path('dashboard/output/final_dashboard.html'),
+)
+"
+```
+
+## Final ranker
+
+1. **Block A — exact:** All jobs passing exact-match thresholds, in exact-match order (`src=exact`). Optional “graph agrees #N” badge.
+2. **Block B — graph:** Dijkstra on jobs not in Block A, capped at 15 (`src=graph`).
+
+No job appears twice. The dashboard **Final** column shows rank order and badges only — no combined score.
+
+## Graph scoring
+
+- Edge weight: `1 + ABSTRACTION_ALPHA × (max_depth - level)` (`ABSTRACTION_ALPHA=1.0` in `graph_engine/build_graph.py`).
+- Per job skill: minimum weighted distance from any user skill node (0 = same node).
+- Sort key: `(-exact_node_matches, avg_distance)`.

backend/experiments/two_step_reterival_skills_graphs/dashboard/run_njila_dashboard.py

@@ -0,0 +1,83 @@
+#!/usr/bin/env python3
+"""Run skills-graph matching for all Njila users and build comparison dashboard."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import logging
+import sys
+import time
+from pathlib import Path
+
+from build_dashboard import write_dashboard
+
+DASHBOARD_DIR = Path(__file__).resolve().parent
+EXPERIMENT_DIR = DASHBOARD_DIR.parent
+OUTPUT_DIR = DASHBOARD_DIR / "output"
+DEFAULT_USERS = EXPERIMENT_DIR / "data" / "njila_users.jsonl"
+DEFAULT_JOBS = EXPERIMENT_DIR / "data" / "ranked_jobs_v2.json"
+
+logging.basicConfig(level=logging.INFO, format="%(levelname)s | %(message)s")
+logger = logging.getLogger(__name__)
+
+
+def main() -> int:
+    sys.path.insert(0, str(EXPERIMENT_DIR))
+    from graph_engine.context import DEFAULT_TAXONOMY_DIR, load_context
+    from graph_engine.user_profile import load_users_jsonl, parse_user
+    from registry import ALL, dashboard_meta, run_all, user_dashboard_record
+
+    parser = argparse.ArgumentParser(description="Run Njila skills-graph dashboard")
+    parser.add_argument("--users", type=Path, default=DEFAULT_USERS)
+    parser.add_argument("--jobs", type=Path, default=DEFAULT_JOBS)
+    parser.add_argument("--taxonomy", type=Path, default=DEFAULT_TAXONOMY_DIR)
+    parser.add_argument(
+        "--json-out", type=Path, default=OUTPUT_DIR / "final_dashboard.json"
+    )
+    parser.add_argument(
+        "--html-out", type=Path, default=OUTPUT_DIR / "final_dashboard.html"
+    )
+    parser.add_argument("--top-n", type=int, default=30)
+    args = parser.parse_args()
+
+    t0 = time.perf_counter()
+    logger.info("Loading taxonomy + %s jobs …", args.jobs.name)
+    ctx = load_context(args.taxonomy, args.jobs)
+    logger.info("Graph: %d nodes · jobs=%d", ctx.tax.number_of_nodes(), len(ctx.jobs))
+
+    users_raw = load_users_jsonl(args.users)
+    logger.info("Loaded %d users from %s", len(users_raw), args.users)
+
+    records: list[dict] = []
+    for i, raw in enumerate(users_raw, 1):
+        user = parse_user(raw)
+        logger.info(
+            "[%d/%d] Matching user %s (%d skills) …",
+            i,
+            len(users_raw),
+            user.user_id,
+            len(user.skills),
+        )
+        rankings = run_all(user, ctx, top_n=args.top_n)
+        records.append(user_dashboard_record(raw, user, rankings))
+
+    payload = {
+        "meta": {"jobs": len(ctx.jobs), **dashboard_meta(len(records))},
+        "data": {mod.NAME: records for mod in ALL},
+    }
+
+    args.json_out.parent.mkdir(parents=True, exist_ok=True)
+    args.json_out.write_text(
+        json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8"
+    )
+    logger.info("Wrote JSON → %s", args.json_out)
+
+    write_dashboard(args.json_out, args.html_out)
+    logger.info("Wrote HTML → %s", args.html_out)
+    logger.info("Done in %.1fs", time.perf_counter() - t0)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())