From aad1481c38608ec0651c4c8ebce78f974362259c Mon Sep 17 00:00:00 2001 From: Andre Brait Date: Thu, 10 Sep 2026 14:51:08 +0000 Subject: [PATCH 1/2] fix: harden generated Graphify instructions --- graphify/skill-agents.md | 52 +++++++++--------- graphify/skill-aider.md | 30 +++++------ graphify/skill-amp.md | 52 +++++++++--------- graphify/skill-claw.md | 52 +++++++++--------- graphify/skill-codex.md | 52 +++++++++--------- graphify/skill-copilot.md | 52 +++++++++--------- graphify/skill-devin.md | 30 +++++------ graphify/skill-droid.md | 52 +++++++++--------- graphify/skill-kilo.md | 52 +++++++++--------- graphify/skill-kiro.md | 52 +++++++++--------- graphify/skill-opencode.md | 52 +++++++++--------- graphify/skill-pi.md | 52 +++++++++--------- graphify/skill-trae.md | 52 +++++++++--------- graphify/skill-vscode.md | 52 +++++++++--------- graphify/skill-windows.md | 52 +++++++++--------- graphify/skill.md | 52 +++++++++--------- .../skills/agents/references/transcribe.md | 6 +-- graphify/skills/agents/references/update.md | 14 ++--- graphify/skills/amp/references/transcribe.md | 6 +-- graphify/skills/amp/references/update.md | 14 ++--- .../skills/claude/references/transcribe.md | 6 +-- graphify/skills/claude/references/update.md | 14 ++--- graphify/skills/claw/references/transcribe.md | 6 +-- graphify/skills/claw/references/update.md | 14 ++--- .../skills/codex/references/transcribe.md | 6 +-- graphify/skills/codex/references/update.md | 14 ++--- .../skills/copilot/references/transcribe.md | 6 +-- graphify/skills/copilot/references/update.md | 14 ++--- .../skills/droid/references/transcribe.md | 6 +-- graphify/skills/droid/references/update.md | 14 ++--- graphify/skills/kilo/references/transcribe.md | 6 +-- graphify/skills/kilo/references/update.md | 14 ++--- graphify/skills/kiro/references/transcribe.md | 6 +-- graphify/skills/kiro/references/update.md | 14 ++--- .../skills/opencode/references/transcribe.md | 6 +-- graphify/skills/opencode/references/update.md | 14 ++--- graphify/skills/pi/references/transcribe.md | 6 +-- graphify/skills/pi/references/update.md | 14 ++--- graphify/skills/trae/references/transcribe.md | 6 +-- graphify/skills/trae/references/update.md | 14 ++--- .../skills/vscode/references/transcribe.md | 6 +-- graphify/skills/vscode/references/update.md | 14 ++--- .../skills/windows/references/transcribe.md | 6 +-- graphify/skills/windows/references/update.md | 14 ++--- tests/test_skillgen.py | 53 +++++++++++++++++-- .../expected/graphify__skill-agents.md | 52 +++++++++--------- .../expected/graphify__skill-aider.md | 30 +++++------ .../skillgen/expected/graphify__skill-amp.md | 52 +++++++++--------- .../skillgen/expected/graphify__skill-claw.md | 52 +++++++++--------- .../expected/graphify__skill-codex.md | 52 +++++++++--------- .../expected/graphify__skill-copilot.md | 52 +++++++++--------- .../expected/graphify__skill-devin.md | 30 +++++------ .../expected/graphify__skill-droid.md | 52 +++++++++--------- .../skillgen/expected/graphify__skill-kilo.md | 52 +++++++++--------- .../skillgen/expected/graphify__skill-kiro.md | 52 +++++++++--------- .../expected/graphify__skill-opencode.md | 52 +++++++++--------- tools/skillgen/expected/graphify__skill-pi.md | 52 +++++++++--------- .../skillgen/expected/graphify__skill-trae.md | 52 +++++++++--------- .../expected/graphify__skill-vscode.md | 52 +++++++++--------- .../expected/graphify__skill-windows.md | 52 +++++++++--------- tools/skillgen/expected/graphify__skill.md | 52 +++++++++--------- ..._skills__agents__references__transcribe.md | 6 +-- ...ify__skills__agents__references__update.md | 14 ++--- ...fy__skills__amp__references__transcribe.md | 6 +-- ...aphify__skills__amp__references__update.md | 14 ++--- ..._skills__claude__references__transcribe.md | 6 +-- ...ify__skills__claude__references__update.md | 14 ++--- ...y__skills__claw__references__transcribe.md | 6 +-- ...phify__skills__claw__references__update.md | 14 ++--- ...__skills__codex__references__transcribe.md | 6 +-- ...hify__skills__codex__references__update.md | 14 ++--- ...skills__copilot__references__transcribe.md | 6 +-- ...fy__skills__copilot__references__update.md | 14 ++--- ...__skills__droid__references__transcribe.md | 6 +-- ...hify__skills__droid__references__update.md | 14 ++--- ...y__skills__kilo__references__transcribe.md | 6 +-- ...phify__skills__kilo__references__update.md | 14 ++--- ...y__skills__kiro__references__transcribe.md | 6 +-- ...phify__skills__kiro__references__update.md | 14 ++--- ...kills__opencode__references__transcribe.md | 6 +-- ...y__skills__opencode__references__update.md | 14 ++--- ...ify__skills__pi__references__transcribe.md | 6 +-- ...raphify__skills__pi__references__update.md | 14 ++--- ...y__skills__trae__references__transcribe.md | 6 +-- ...phify__skills__trae__references__update.md | 14 ++--- ..._skills__vscode__references__transcribe.md | 6 +-- ...ify__skills__vscode__references__update.md | 14 ++--- ...skills__windows__references__transcribe.md | 6 +-- ...fy__skills__windows__references__update.md | 14 ++--- tools/skillgen/fragments/core/aider.md | 30 +++++------ tools/skillgen/fragments/core/core.md | 52 +++++++++--------- tools/skillgen/fragments/core/devin.md | 30 +++++------ .../fragments/references/shared/transcribe.md | 6 +-- .../fragments/references/shared/update.md | 14 ++--- tools/skillgen/gen.py | 43 ++++++++++++++- 95 files changed, 1195 insertions(+), 1169 deletions(-) diff --git a/graphify/skill-agents.md b/graphify/skill-agents.md index 846a316445..78361e1bde 100644 --- a/graphify/skill-agents.md +++ b/graphify/skill-agents.md @@ -108,15 +108,15 @@ If the import succeeds, print nothing and move straight to Step 2. ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from graphify.detect import detect from pathlib import Path -result = detect(Path('INPUT_PATH')) +result = detect(Path(sys.argv[1])) # Write the sidecar from Python, not a shell redirect, so the same block renders # on PowerShell hosts without console-encoding drift (#2528). Path('graphify-out/.graphify_detect.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") print(f'Detected {result[\"total_files\"]} files') -" +" "INPUT_PATH" ``` Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: @@ -183,13 +183,13 @@ for f in detect.get('files', {}).get('code', []): code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) if code_files: - result = extract(code_files, cache_root=Path('INPUT_PATH')) + result = extract(code_files, cache_root=Path(sys.argv[1])) Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') else: Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") print('No code files - skipping AST extraction') -" +" "INPUT_PATH" ``` #### Part B - Semantic extraction (parallel subagents) @@ -204,7 +204,7 @@ Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],' " ``` -**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** +**MANDATORY: You MUST use the subagent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the subagent tool you are doing this wrong.** Before dispatching subagents, print a timing estimate: - Load `total_words` and file counts from `graphify-out/.graphify_detect.json` @@ -220,7 +220,7 @@ SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from graphify.cache import check_semantic_cache from pathlib import Path @@ -230,7 +230,7 @@ detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encodin # every source file (#1392). Video is transcribed to a document in Step 2.5 first. all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root=sys.argv[1], prompt_file=sys.argv[2]) # Always (re)write the cache file: write hits, else DELETE any leftover from a prior # run so Part C never merges a stale .graphify_cached.json (#1392). @@ -240,7 +240,7 @@ else: Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" +" "INPUT_PATH" "SPEC_PATH" ``` Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. @@ -282,7 +282,7 @@ Wait for all subagents. For each result: If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: +Merge all chunk files into `.graphify_semantic_new.json`. **After each subagent call completes, read the real token counts from the subagent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: ```bash $(cat graphify-out/.graphify_python) -c " import json, glob @@ -309,15 +309,15 @@ print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens' Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root=sys.argv[1], allowed_source_files=uncached, prompt_file=sys.argv[2]) print(f'Cached {saved} files') -" +" "INPUT_PATH" "SPEC_PATH" ``` Merge cached + new results into `graphify-out/.graphify_semantic.json`: @@ -406,7 +406,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -431,7 +431,7 @@ if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') print('If this shrink is intentional (you deleted files), re-run a full build with --force.') raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, sys.argv[1], suggested_questions=questions) Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") analysis = { 'communities': {str(k): v for k, v in communities.items()}, @@ -442,7 +442,7 @@ analysis = { } Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" +" "INPUT_PATH" ``` If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. @@ -455,12 +455,12 @@ A non-destructive diagnostic on the extraction, before labeling. It surfaces edg ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from pathlib import Path from graphify.diagnostics import diagnose_extraction, format_diagnostic_report extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') +summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root=sys.argv[1]) print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( ('dangling_endpoint_edges', 'dangling-endpoint edges'), @@ -470,7 +470,7 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0)] print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') -" +" "INPUT_PATH" ``` Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules). @@ -496,7 +496,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} @@ -507,7 +507,7 @@ labels = LABELS_DICT # Regenerate questions with real community labels (labels affect question phrasing) questions = suggest_questions(G, communities, labels) -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, sys.argv[1], suggested_questions=questions) # Re-export so graph.json nodes carry the curated community_name (#2490). # Same extraction as Step 4, so the #479 shrink-guard passes on node count; # if it still refuses, surface the guard message - do not force past it. @@ -519,7 +519,7 @@ if not wrote: Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") print('Report updated with community labels') -" +" "INPUT_PATH" ``` Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). @@ -554,7 +554,7 @@ These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, ` ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from pathlib import Path from datetime import datetime, timezone from graphify.detect import save_manifest @@ -577,7 +577,7 @@ extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encod # types are gated on output. from graphify.cli import _stamped_manifest_files _corpus = detect.get('all_files') or detect['files'] -_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) +_manifest_files = _stamped_manifest_files(_corpus, extract, Path(sys.argv[1])) # Files dispatched this run (the changed subset) but NOT stamped above still carry # a stale semantic_hash from a prior run; clear it so detect_incremental re-queues # them instead of reading them as unchanged (#1948). @@ -589,7 +589,7 @@ _cleared = _dispatched - _stamped # files newly excluded since last run are dropped rather than masquerading as # deletions; untouched files' prior rows are still preserved (#1908). _scan = {f for fl in _corpus.values() for f in fl} -save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) # Update cumulative cost tracker input_tok = extract.get('input_tokens', 0) @@ -613,7 +613,7 @@ cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\" print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" +" "INPUT_PATH" rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null rm -f graphify-out/needs_update 2>/dev/null || true diff --git a/graphify/skill-aider.md b/graphify/skill-aider.md index 4c5f74c88b..085be97cd2 100644 --- a/graphify/skill-aider.md +++ b/graphify/skill-aider.md @@ -101,11 +101,12 @@ If the import succeeds, print nothing and move straight to Step 2. ```bash $(cat graphify-out/.graphify_python) -c " import json +import sys from graphify.detect import detect from pathlib import Path -result = detect(Path('INPUT_PATH')) +result = detect(Path(sys.argv[1])) print(json.dumps(result)) -" > .graphify_detect.json +" "INPUT_PATH" > .graphify_detect.json ``` Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: @@ -133,13 +134,11 @@ Skip this step entirely if `detect` returned zero `video` files. Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. -**Strategy:** Read the god nodes from the detect output or analysis file. You are already a language model - write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. - -**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` +**Strategy:** If an analysis file exists from a previous run, read its top god-node labels and write a one-sentence domain hint from them. Otherwise use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` Pass the chosen prompt to Whisper as its initial prompt. No separate API call needed. **Step 1 - Write the Whisper prompt yourself.** -Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: +Read the top god-node labels from previous analysis when available, then compose a short domain hint sentence. With no previous analysis, use the generic fallback above. - Labels: `transformer, attention, encoder, decoder` -> `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` - Labels: `kubernetes, deployment, pod, helm` -> `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` @@ -426,7 +425,7 @@ wrote = to_json(G, communities, 'graphify-out/graph.json') if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (fewer nodes than the existing graph). Run a full rebuild to be safe.') raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, sys.argv[1], suggested_questions=questions) Path('graphify-out/GRAPH_REPORT.md').write_text(report) analysis = { @@ -438,7 +437,7 @@ analysis = { } Path('.graphify_analysis.json').write_text(json.dumps(analysis, indent=2)) print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" +" "INPUT_PATH" ``` If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. @@ -476,7 +475,7 @@ labels = LABELS_DICT # Regenerate questions with real community labels (labels affect question phrasing) questions = suggest_questions(G, communities, labels) -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, sys.argv[1], suggested_questions=questions) # Re-export so graph.json nodes carry the curated community_name (#2490). # Same extraction as Step 4, so the #479 shrink-guard passes on node count; # if it still refuses, surface the guard message - do not force past it. @@ -487,7 +486,7 @@ if not wrote: Path('graphify-out/GRAPH_REPORT.md').write_text(report) Path('.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()})) print('Report updated with community labels') -" +" "INPUT_PATH" ``` Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). @@ -679,6 +678,7 @@ Print the output directly in chat. If `total_words <= 5000`, skip silently - the ```bash $(cat graphify-out/.graphify_python) -c " import json +import sys from pathlib import Path from datetime import datetime, timezone from graphify.detect import save_manifest @@ -689,13 +689,13 @@ extract = json.loads(Path('.graphify_extract.json').read_text()) # Stamp only semantic files that produced output so a failed chunk is re-queued next run, not lost (#2015). from graphify.cli import _stamped_manifest_files _corpus = detect.get('all_files') or detect['files'] -_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) +_manifest_files = _stamped_manifest_files(_corpus, extract, Path(sys.argv[1])) _sem_types = ('document', 'paper', 'image') _dispatched = {f for t, fl in detect['files'].items() if t in _sem_types for f in fl} _stamped = {f for fl in _manifest_files.values() for f in fl} _cleared = _dispatched - _stamped _scan = {f for fl in _corpus.values() for f in fl} -save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) # Update cumulative cost tracker input_tok = extract.get('input_tokens', 0) @@ -719,7 +719,7 @@ cost_path.write_text(json.dumps(cost, indent=2)) print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" +" "INPUT_PATH" rm -f .graphify_detect.json .graphify_extract.json .graphify_ast.json .graphify_semantic.json .graphify_analysis.json .graphify_labels.json; find . -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null rm -f graphify-out/needs_update 2>/dev/null || true ``` @@ -765,7 +765,7 @@ import sys, json from graphify.detect import detect_incremental, save_manifest from pathlib import Path -result = detect_incremental(Path('INPUT_PATH')) +result = detect_incremental(Path(sys.argv[1])) new_total = result.get('new_total', 0) print(json.dumps(result, indent=2)) Path('.graphify_incremental.json').write_text(json.dumps(result)) @@ -777,7 +777,7 @@ if deleted: print(f'{len(deleted)} deleted file(s) to prune.') if new_total > 0: print(f'{new_total} new/changed file(s) to re-extract.') -" +" "INPUT_PATH" ``` If new files exist, first check whether all changed files are code files: diff --git a/graphify/skill-amp.md b/graphify/skill-amp.md index 846a316445..78361e1bde 100644 --- a/graphify/skill-amp.md +++ b/graphify/skill-amp.md @@ -108,15 +108,15 @@ If the import succeeds, print nothing and move straight to Step 2. ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from graphify.detect import detect from pathlib import Path -result = detect(Path('INPUT_PATH')) +result = detect(Path(sys.argv[1])) # Write the sidecar from Python, not a shell redirect, so the same block renders # on PowerShell hosts without console-encoding drift (#2528). Path('graphify-out/.graphify_detect.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") print(f'Detected {result[\"total_files\"]} files') -" +" "INPUT_PATH" ``` Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: @@ -183,13 +183,13 @@ for f in detect.get('files', {}).get('code', []): code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) if code_files: - result = extract(code_files, cache_root=Path('INPUT_PATH')) + result = extract(code_files, cache_root=Path(sys.argv[1])) Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') else: Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") print('No code files - skipping AST extraction') -" +" "INPUT_PATH" ``` #### Part B - Semantic extraction (parallel subagents) @@ -204,7 +204,7 @@ Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],' " ``` -**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** +**MANDATORY: You MUST use the subagent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the subagent tool you are doing this wrong.** Before dispatching subagents, print a timing estimate: - Load `total_words` and file counts from `graphify-out/.graphify_detect.json` @@ -220,7 +220,7 @@ SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from graphify.cache import check_semantic_cache from pathlib import Path @@ -230,7 +230,7 @@ detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encodin # every source file (#1392). Video is transcribed to a document in Step 2.5 first. all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root=sys.argv[1], prompt_file=sys.argv[2]) # Always (re)write the cache file: write hits, else DELETE any leftover from a prior # run so Part C never merges a stale .graphify_cached.json (#1392). @@ -240,7 +240,7 @@ else: Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" +" "INPUT_PATH" "SPEC_PATH" ``` Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. @@ -282,7 +282,7 @@ Wait for all subagents. For each result: If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: +Merge all chunk files into `.graphify_semantic_new.json`. **After each subagent call completes, read the real token counts from the subagent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: ```bash $(cat graphify-out/.graphify_python) -c " import json, glob @@ -309,15 +309,15 @@ print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens' Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root=sys.argv[1], allowed_source_files=uncached, prompt_file=sys.argv[2]) print(f'Cached {saved} files') -" +" "INPUT_PATH" "SPEC_PATH" ``` Merge cached + new results into `graphify-out/.graphify_semantic.json`: @@ -406,7 +406,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -431,7 +431,7 @@ if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') print('If this shrink is intentional (you deleted files), re-run a full build with --force.') raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, sys.argv[1], suggested_questions=questions) Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") analysis = { 'communities': {str(k): v for k, v in communities.items()}, @@ -442,7 +442,7 @@ analysis = { } Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" +" "INPUT_PATH" ``` If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. @@ -455,12 +455,12 @@ A non-destructive diagnostic on the extraction, before labeling. It surfaces edg ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from pathlib import Path from graphify.diagnostics import diagnose_extraction, format_diagnostic_report extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') +summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root=sys.argv[1]) print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( ('dangling_endpoint_edges', 'dangling-endpoint edges'), @@ -470,7 +470,7 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0)] print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') -" +" "INPUT_PATH" ``` Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules). @@ -496,7 +496,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} @@ -507,7 +507,7 @@ labels = LABELS_DICT # Regenerate questions with real community labels (labels affect question phrasing) questions = suggest_questions(G, communities, labels) -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, sys.argv[1], suggested_questions=questions) # Re-export so graph.json nodes carry the curated community_name (#2490). # Same extraction as Step 4, so the #479 shrink-guard passes on node count; # if it still refuses, surface the guard message - do not force past it. @@ -519,7 +519,7 @@ if not wrote: Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") print('Report updated with community labels') -" +" "INPUT_PATH" ``` Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). @@ -554,7 +554,7 @@ These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, ` ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from pathlib import Path from datetime import datetime, timezone from graphify.detect import save_manifest @@ -577,7 +577,7 @@ extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encod # types are gated on output. from graphify.cli import _stamped_manifest_files _corpus = detect.get('all_files') or detect['files'] -_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) +_manifest_files = _stamped_manifest_files(_corpus, extract, Path(sys.argv[1])) # Files dispatched this run (the changed subset) but NOT stamped above still carry # a stale semantic_hash from a prior run; clear it so detect_incremental re-queues # them instead of reading them as unchanged (#1948). @@ -589,7 +589,7 @@ _cleared = _dispatched - _stamped # files newly excluded since last run are dropped rather than masquerading as # deletions; untouched files' prior rows are still preserved (#1908). _scan = {f for fl in _corpus.values() for f in fl} -save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) # Update cumulative cost tracker input_tok = extract.get('input_tokens', 0) @@ -613,7 +613,7 @@ cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\" print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" +" "INPUT_PATH" rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null rm -f graphify-out/needs_update 2>/dev/null || true diff --git a/graphify/skill-claw.md b/graphify/skill-claw.md index 56c5c75fc2..7e7c797563 100644 --- a/graphify/skill-claw.md +++ b/graphify/skill-claw.md @@ -108,15 +108,15 @@ If the import succeeds, print nothing and move straight to Step 2. ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from graphify.detect import detect from pathlib import Path -result = detect(Path('INPUT_PATH')) +result = detect(Path(sys.argv[1])) # Write the sidecar from Python, not a shell redirect, so the same block renders # on PowerShell hosts without console-encoding drift (#2528). Path('graphify-out/.graphify_detect.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") print(f'Detected {result[\"total_files\"]} files') -" +" "INPUT_PATH" ``` Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: @@ -183,13 +183,13 @@ for f in detect.get('files', {}).get('code', []): code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) if code_files: - result = extract(code_files, cache_root=Path('INPUT_PATH')) + result = extract(code_files, cache_root=Path(sys.argv[1])) Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') else: Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") print('No code files - skipping AST extraction') -" +" "INPUT_PATH" ``` #### Part B - Semantic extraction (parallel subagents) @@ -204,7 +204,7 @@ Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],' " ``` -**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** +**MANDATORY: You MUST use the subagent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the subagent tool you are doing this wrong.** Before dispatching subagents, print a timing estimate: - Load `total_words` and file counts from `graphify-out/.graphify_detect.json` @@ -220,7 +220,7 @@ SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from graphify.cache import check_semantic_cache from pathlib import Path @@ -230,7 +230,7 @@ detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encodin # every source file (#1392). Video is transcribed to a document in Step 2.5 first. all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root=sys.argv[1], prompt_file=sys.argv[2]) # Always (re)write the cache file: write hits, else DELETE any leftover from a prior # run so Part C never merges a stale .graphify_cached.json (#1392). @@ -240,7 +240,7 @@ else: Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" +" "INPUT_PATH" "SPEC_PATH" ``` Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. @@ -285,7 +285,7 @@ Wait for all subagents. For each result: If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: +Merge all chunk files into `.graphify_semantic_new.json`. **After each subagent call completes, read the real token counts from the subagent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: ```bash $(cat graphify-out/.graphify_python) -c " import json, glob @@ -312,15 +312,15 @@ print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens' Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root=sys.argv[1], allowed_source_files=uncached, prompt_file=sys.argv[2]) print(f'Cached {saved} files') -" +" "INPUT_PATH" "SPEC_PATH" ``` Merge cached + new results into `graphify-out/.graphify_semantic.json`: @@ -409,7 +409,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -434,7 +434,7 @@ if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') print('If this shrink is intentional (you deleted files), re-run a full build with --force.') raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, sys.argv[1], suggested_questions=questions) Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") analysis = { 'communities': {str(k): v for k, v in communities.items()}, @@ -445,7 +445,7 @@ analysis = { } Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" +" "INPUT_PATH" ``` If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. @@ -458,12 +458,12 @@ A non-destructive diagnostic on the extraction, before labeling. It surfaces edg ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from pathlib import Path from graphify.diagnostics import diagnose_extraction, format_diagnostic_report extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') +summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root=sys.argv[1]) print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( ('dangling_endpoint_edges', 'dangling-endpoint edges'), @@ -473,7 +473,7 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0)] print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') -" +" "INPUT_PATH" ``` Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules). @@ -499,7 +499,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} @@ -510,7 +510,7 @@ labels = LABELS_DICT # Regenerate questions with real community labels (labels affect question phrasing) questions = suggest_questions(G, communities, labels) -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, sys.argv[1], suggested_questions=questions) # Re-export so graph.json nodes carry the curated community_name (#2490). # Same extraction as Step 4, so the #479 shrink-guard passes on node count; # if it still refuses, surface the guard message - do not force past it. @@ -522,7 +522,7 @@ if not wrote: Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") print('Report updated with community labels') -" +" "INPUT_PATH" ``` Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). @@ -557,7 +557,7 @@ These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, ` ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from pathlib import Path from datetime import datetime, timezone from graphify.detect import save_manifest @@ -580,7 +580,7 @@ extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encod # types are gated on output. from graphify.cli import _stamped_manifest_files _corpus = detect.get('all_files') or detect['files'] -_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) +_manifest_files = _stamped_manifest_files(_corpus, extract, Path(sys.argv[1])) # Files dispatched this run (the changed subset) but NOT stamped above still carry # a stale semantic_hash from a prior run; clear it so detect_incremental re-queues # them instead of reading them as unchanged (#1948). @@ -592,7 +592,7 @@ _cleared = _dispatched - _stamped # files newly excluded since last run are dropped rather than masquerading as # deletions; untouched files' prior rows are still preserved (#1908). _scan = {f for fl in _corpus.values() for f in fl} -save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) # Update cumulative cost tracker input_tok = extract.get('input_tokens', 0) @@ -616,7 +616,7 @@ cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\" print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" +" "INPUT_PATH" rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null rm -f graphify-out/needs_update 2>/dev/null || true diff --git a/graphify/skill-codex.md b/graphify/skill-codex.md index 4b23633542..5e085f0a6f 100644 --- a/graphify/skill-codex.md +++ b/graphify/skill-codex.md @@ -108,15 +108,15 @@ If the import succeeds, print nothing and move straight to Step 2. ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from graphify.detect import detect from pathlib import Path -result = detect(Path('INPUT_PATH')) +result = detect(Path(sys.argv[1])) # Write the sidecar from Python, not a shell redirect, so the same block renders # on PowerShell hosts without console-encoding drift (#2528). Path('graphify-out/.graphify_detect.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") print(f'Detected {result[\"total_files\"]} files') -" +" "INPUT_PATH" ``` Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: @@ -183,13 +183,13 @@ for f in detect.get('files', {}).get('code', []): code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) if code_files: - result = extract(code_files, cache_root=Path('INPUT_PATH')) + result = extract(code_files, cache_root=Path(sys.argv[1])) Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') else: Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") print('No code files - skipping AST extraction') -" +" "INPUT_PATH" ``` #### Part B - Semantic extraction (parallel subagents) @@ -204,7 +204,7 @@ Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],' " ``` -**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** +**MANDATORY: You MUST use the subagent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the subagent tool you are doing this wrong.** Before dispatching subagents, print a timing estimate: - Load `total_words` and file counts from `graphify-out/.graphify_detect.json` @@ -220,7 +220,7 @@ SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from graphify.cache import check_semantic_cache from pathlib import Path @@ -230,7 +230,7 @@ detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encodin # every source file (#1392). Video is transcribed to a document in Step 2.5 first. all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root=sys.argv[1], prompt_file=sys.argv[2]) # Always (re)write the cache file: write hits, else DELETE any leftover from a prior # run so Part C never merges a stale .graphify_cached.json (#1392). @@ -240,7 +240,7 @@ else: Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" +" "INPUT_PATH" "SPEC_PATH" ``` Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. @@ -282,7 +282,7 @@ Wait for all subagents. For each result: If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: +Merge all chunk files into `.graphify_semantic_new.json`. **After each subagent call completes, read the real token counts from the subagent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: ```bash $(cat graphify-out/.graphify_python) -c " import json, glob @@ -309,15 +309,15 @@ print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens' Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root=sys.argv[1], allowed_source_files=uncached, prompt_file=sys.argv[2]) print(f'Cached {saved} files') -" +" "INPUT_PATH" "SPEC_PATH" ``` Merge cached + new results into `graphify-out/.graphify_semantic.json`: @@ -406,7 +406,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -431,7 +431,7 @@ if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') print('If this shrink is intentional (you deleted files), re-run a full build with --force.') raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, sys.argv[1], suggested_questions=questions) Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") analysis = { 'communities': {str(k): v for k, v in communities.items()}, @@ -442,7 +442,7 @@ analysis = { } Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" +" "INPUT_PATH" ``` If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. @@ -455,12 +455,12 @@ A non-destructive diagnostic on the extraction, before labeling. It surfaces edg ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from pathlib import Path from graphify.diagnostics import diagnose_extraction, format_diagnostic_report extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') +summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root=sys.argv[1]) print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( ('dangling_endpoint_edges', 'dangling-endpoint edges'), @@ -470,7 +470,7 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0)] print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') -" +" "INPUT_PATH" ``` Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules). @@ -496,7 +496,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} @@ -507,7 +507,7 @@ labels = LABELS_DICT # Regenerate questions with real community labels (labels affect question phrasing) questions = suggest_questions(G, communities, labels) -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, sys.argv[1], suggested_questions=questions) # Re-export so graph.json nodes carry the curated community_name (#2490). # Same extraction as Step 4, so the #479 shrink-guard passes on node count; # if it still refuses, surface the guard message - do not force past it. @@ -519,7 +519,7 @@ if not wrote: Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") print('Report updated with community labels') -" +" "INPUT_PATH" ``` Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). @@ -554,7 +554,7 @@ These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, ` ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from pathlib import Path from datetime import datetime, timezone from graphify.detect import save_manifest @@ -577,7 +577,7 @@ extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encod # types are gated on output. from graphify.cli import _stamped_manifest_files _corpus = detect.get('all_files') or detect['files'] -_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) +_manifest_files = _stamped_manifest_files(_corpus, extract, Path(sys.argv[1])) # Files dispatched this run (the changed subset) but NOT stamped above still carry # a stale semantic_hash from a prior run; clear it so detect_incremental re-queues # them instead of reading them as unchanged (#1948). @@ -589,7 +589,7 @@ _cleared = _dispatched - _stamped # files newly excluded since last run are dropped rather than masquerading as # deletions; untouched files' prior rows are still preserved (#1908). _scan = {f for fl in _corpus.values() for f in fl} -save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) # Update cumulative cost tracker input_tok = extract.get('input_tokens', 0) @@ -613,7 +613,7 @@ cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\" print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" +" "INPUT_PATH" rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null rm -f graphify-out/needs_update 2>/dev/null || true diff --git a/graphify/skill-copilot.md b/graphify/skill-copilot.md index 56c5c75fc2..7e7c797563 100644 --- a/graphify/skill-copilot.md +++ b/graphify/skill-copilot.md @@ -108,15 +108,15 @@ If the import succeeds, print nothing and move straight to Step 2. ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from graphify.detect import detect from pathlib import Path -result = detect(Path('INPUT_PATH')) +result = detect(Path(sys.argv[1])) # Write the sidecar from Python, not a shell redirect, so the same block renders # on PowerShell hosts without console-encoding drift (#2528). Path('graphify-out/.graphify_detect.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") print(f'Detected {result[\"total_files\"]} files') -" +" "INPUT_PATH" ``` Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: @@ -183,13 +183,13 @@ for f in detect.get('files', {}).get('code', []): code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) if code_files: - result = extract(code_files, cache_root=Path('INPUT_PATH')) + result = extract(code_files, cache_root=Path(sys.argv[1])) Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') else: Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") print('No code files - skipping AST extraction') -" +" "INPUT_PATH" ``` #### Part B - Semantic extraction (parallel subagents) @@ -204,7 +204,7 @@ Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],' " ``` -**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** +**MANDATORY: You MUST use the subagent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the subagent tool you are doing this wrong.** Before dispatching subagents, print a timing estimate: - Load `total_words` and file counts from `graphify-out/.graphify_detect.json` @@ -220,7 +220,7 @@ SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from graphify.cache import check_semantic_cache from pathlib import Path @@ -230,7 +230,7 @@ detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encodin # every source file (#1392). Video is transcribed to a document in Step 2.5 first. all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root=sys.argv[1], prompt_file=sys.argv[2]) # Always (re)write the cache file: write hits, else DELETE any leftover from a prior # run so Part C never merges a stale .graphify_cached.json (#1392). @@ -240,7 +240,7 @@ else: Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" +" "INPUT_PATH" "SPEC_PATH" ``` Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. @@ -285,7 +285,7 @@ Wait for all subagents. For each result: If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: +Merge all chunk files into `.graphify_semantic_new.json`. **After each subagent call completes, read the real token counts from the subagent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: ```bash $(cat graphify-out/.graphify_python) -c " import json, glob @@ -312,15 +312,15 @@ print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens' Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root=sys.argv[1], allowed_source_files=uncached, prompt_file=sys.argv[2]) print(f'Cached {saved} files') -" +" "INPUT_PATH" "SPEC_PATH" ``` Merge cached + new results into `graphify-out/.graphify_semantic.json`: @@ -409,7 +409,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -434,7 +434,7 @@ if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') print('If this shrink is intentional (you deleted files), re-run a full build with --force.') raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, sys.argv[1], suggested_questions=questions) Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") analysis = { 'communities': {str(k): v for k, v in communities.items()}, @@ -445,7 +445,7 @@ analysis = { } Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" +" "INPUT_PATH" ``` If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. @@ -458,12 +458,12 @@ A non-destructive diagnostic on the extraction, before labeling. It surfaces edg ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from pathlib import Path from graphify.diagnostics import diagnose_extraction, format_diagnostic_report extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') +summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root=sys.argv[1]) print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( ('dangling_endpoint_edges', 'dangling-endpoint edges'), @@ -473,7 +473,7 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0)] print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') -" +" "INPUT_PATH" ``` Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules). @@ -499,7 +499,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} @@ -510,7 +510,7 @@ labels = LABELS_DICT # Regenerate questions with real community labels (labels affect question phrasing) questions = suggest_questions(G, communities, labels) -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, sys.argv[1], suggested_questions=questions) # Re-export so graph.json nodes carry the curated community_name (#2490). # Same extraction as Step 4, so the #479 shrink-guard passes on node count; # if it still refuses, surface the guard message - do not force past it. @@ -522,7 +522,7 @@ if not wrote: Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") print('Report updated with community labels') -" +" "INPUT_PATH" ``` Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). @@ -557,7 +557,7 @@ These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, ` ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from pathlib import Path from datetime import datetime, timezone from graphify.detect import save_manifest @@ -580,7 +580,7 @@ extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encod # types are gated on output. from graphify.cli import _stamped_manifest_files _corpus = detect.get('all_files') or detect['files'] -_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) +_manifest_files = _stamped_manifest_files(_corpus, extract, Path(sys.argv[1])) # Files dispatched this run (the changed subset) but NOT stamped above still carry # a stale semantic_hash from a prior run; clear it so detect_incremental re-queues # them instead of reading them as unchanged (#1948). @@ -592,7 +592,7 @@ _cleared = _dispatched - _stamped # files newly excluded since last run are dropped rather than masquerading as # deletions; untouched files' prior rows are still preserved (#1908). _scan = {f for fl in _corpus.values() for f in fl} -save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) # Update cumulative cost tracker input_tok = extract.get('input_tokens', 0) @@ -616,7 +616,7 @@ cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\" print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" +" "INPUT_PATH" rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null rm -f graphify-out/needs_update 2>/dev/null || true diff --git a/graphify/skill-devin.md b/graphify/skill-devin.md index cfb5aa7b13..bea72f7eea 100644 --- a/graphify/skill-devin.md +++ b/graphify/skill-devin.md @@ -114,11 +114,12 @@ If the import succeeds, print nothing and move straight to Step 2. ```bash $(cat graphify-out/.graphify_python) -c " import json +import sys from graphify.detect import detect from pathlib import Path -result = detect(Path('INPUT_PATH')) +result = detect(Path(sys.argv[1])) print(json.dumps(result)) -" > graphify-out/.graphify_detect.json +" "INPUT_PATH" > graphify-out/.graphify_detect.json ``` Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: @@ -146,13 +147,11 @@ Skip this step entirely if `detect` returned zero `video` files. Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. -**Strategy:** Read the god nodes from the detect output or analysis file. You are already a language model - write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. - -**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` +**Strategy:** If an analysis file exists from a previous run, read its top god-node labels and write a one-sentence domain hint from them. Otherwise use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` Pass the chosen prompt to Whisper as its initial prompt. No separate API call needed. **Step 1 - Write the Whisper prompt yourself.** -Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: +Read the top god-node labels from previous analysis when available, then compose a short domain hint sentence. With no previous analysis, use the generic fallback above. - Labels: `transformer, attention, encoder, decoder` -> `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` - Labels: `kubernetes, deployment, pod, helm` -> `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` @@ -491,7 +490,7 @@ wrote = to_json(G, communities, 'graphify-out/graph.json') if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (fewer nodes than the existing graph). Run a full rebuild to be safe.') raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, sys.argv[1], suggested_questions=questions) Path('graphify-out/GRAPH_REPORT.md').write_text(report) analysis = { @@ -503,7 +502,7 @@ analysis = { } Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2)) print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" +" "INPUT_PATH" ``` If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. @@ -541,7 +540,7 @@ labels = LABELS_DICT # Regenerate questions with real community labels (labels affect question phrasing) questions = suggest_questions(G, communities, labels) -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, sys.argv[1], suggested_questions=questions) # Re-export so graph.json nodes carry the curated community_name (#2490). # Same extraction as Step 4, so the #479 shrink-guard passes on node count; # if it still refuses, surface the guard message - do not force past it. @@ -552,7 +551,7 @@ if not wrote: Path('graphify-out/GRAPH_REPORT.md').write_text(report) Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()})) print('Report updated with community labels') -" +" "INPUT_PATH" ``` Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). @@ -797,6 +796,7 @@ Print the output directly in chat. If `total_words <= 5000`, skip silently - the ```bash $(cat graphify-out/.graphify_python) -c " import json +import sys from pathlib import Path from datetime import datetime, timezone from graphify.detect import save_manifest @@ -807,13 +807,13 @@ extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) # Stamp only semantic files that produced output so a failed chunk is re-queued next run, not lost (#2015). from graphify.cli import _stamped_manifest_files _corpus = detect.get('all_files') or detect['files'] -_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) +_manifest_files = _stamped_manifest_files(_corpus, extract, Path(sys.argv[1])) _sem_types = ('document', 'paper', 'image') _dispatched = {f for t, fl in detect['files'].items() if t in _sem_types for f in fl} _stamped = {f for fl in _manifest_files.values() for f in fl} _cleared = _dispatched - _stamped _scan = {f for fl in _corpus.values() for f in fl} -save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) # Update cumulative cost tracker input_tok = extract.get('input_tokens', 0) @@ -837,7 +837,7 @@ cost_path.write_text(json.dumps(cost, indent=2)) print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" +" "INPUT_PATH" rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json graphify-out/.graphify_labels.json graphify-out/.graphify_incremental.json graphify-out/.graphify_transcripts.json graphify-out/.graphify_old.json; find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null rm -f graphify-out/needs_update 2>/dev/null || true ``` @@ -904,7 +904,7 @@ import sys, json from graphify.detect import detect_incremental, save_manifest from pathlib import Path -result = detect_incremental(Path('INPUT_PATH')) +result = detect_incremental(Path(sys.argv[1])) new_total = result.get('new_total', 0) print(json.dumps(result, indent=2)) Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result)) @@ -916,7 +916,7 @@ if deleted: print(f'{len(deleted)} deleted file(s) to prune.') if new_total > 0: print(f'{new_total} new/changed file(s) to re-extract.') -" +" "INPUT_PATH" ``` If new files exist, first check whether all changed files are code files: diff --git a/graphify/skill-droid.md b/graphify/skill-droid.md index f32aac733d..2a896a6fa1 100644 --- a/graphify/skill-droid.md +++ b/graphify/skill-droid.md @@ -108,15 +108,15 @@ If the import succeeds, print nothing and move straight to Step 2. ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from graphify.detect import detect from pathlib import Path -result = detect(Path('INPUT_PATH')) +result = detect(Path(sys.argv[1])) # Write the sidecar from Python, not a shell redirect, so the same block renders # on PowerShell hosts without console-encoding drift (#2528). Path('graphify-out/.graphify_detect.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") print(f'Detected {result[\"total_files\"]} files') -" +" "INPUT_PATH" ``` Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: @@ -183,13 +183,13 @@ for f in detect.get('files', {}).get('code', []): code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) if code_files: - result = extract(code_files, cache_root=Path('INPUT_PATH')) + result = extract(code_files, cache_root=Path(sys.argv[1])) Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') else: Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") print('No code files - skipping AST extraction') -" +" "INPUT_PATH" ``` #### Part B - Semantic extraction (parallel subagents) @@ -204,7 +204,7 @@ Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],' " ``` -**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** +**MANDATORY: You MUST use the subagent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the subagent tool you are doing this wrong.** Before dispatching subagents, print a timing estimate: - Load `total_words` and file counts from `graphify-out/.graphify_detect.json` @@ -220,7 +220,7 @@ SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from graphify.cache import check_semantic_cache from pathlib import Path @@ -230,7 +230,7 @@ detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encodin # every source file (#1392). Video is transcribed to a document in Step 2.5 first. all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root=sys.argv[1], prompt_file=sys.argv[2]) # Always (re)write the cache file: write hits, else DELETE any leftover from a prior # run so Part C never merges a stale .graphify_cached.json (#1392). @@ -240,7 +240,7 @@ else: Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" +" "INPUT_PATH" "SPEC_PATH" ``` Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. @@ -282,7 +282,7 @@ Wait for all subagents. For each result: If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: +Merge all chunk files into `.graphify_semantic_new.json`. **After each subagent call completes, read the real token counts from the subagent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: ```bash $(cat graphify-out/.graphify_python) -c " import json, glob @@ -309,15 +309,15 @@ print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens' Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root=sys.argv[1], allowed_source_files=uncached, prompt_file=sys.argv[2]) print(f'Cached {saved} files') -" +" "INPUT_PATH" "SPEC_PATH" ``` Merge cached + new results into `graphify-out/.graphify_semantic.json`: @@ -406,7 +406,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -431,7 +431,7 @@ if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') print('If this shrink is intentional (you deleted files), re-run a full build with --force.') raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, sys.argv[1], suggested_questions=questions) Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") analysis = { 'communities': {str(k): v for k, v in communities.items()}, @@ -442,7 +442,7 @@ analysis = { } Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" +" "INPUT_PATH" ``` If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. @@ -455,12 +455,12 @@ A non-destructive diagnostic on the extraction, before labeling. It surfaces edg ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from pathlib import Path from graphify.diagnostics import diagnose_extraction, format_diagnostic_report extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') +summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root=sys.argv[1]) print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( ('dangling_endpoint_edges', 'dangling-endpoint edges'), @@ -470,7 +470,7 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0)] print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') -" +" "INPUT_PATH" ``` Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules). @@ -496,7 +496,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} @@ -507,7 +507,7 @@ labels = LABELS_DICT # Regenerate questions with real community labels (labels affect question phrasing) questions = suggest_questions(G, communities, labels) -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, sys.argv[1], suggested_questions=questions) # Re-export so graph.json nodes carry the curated community_name (#2490). # Same extraction as Step 4, so the #479 shrink-guard passes on node count; # if it still refuses, surface the guard message - do not force past it. @@ -519,7 +519,7 @@ if not wrote: Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") print('Report updated with community labels') -" +" "INPUT_PATH" ``` Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). @@ -554,7 +554,7 @@ These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, ` ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from pathlib import Path from datetime import datetime, timezone from graphify.detect import save_manifest @@ -577,7 +577,7 @@ extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encod # types are gated on output. from graphify.cli import _stamped_manifest_files _corpus = detect.get('all_files') or detect['files'] -_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) +_manifest_files = _stamped_manifest_files(_corpus, extract, Path(sys.argv[1])) # Files dispatched this run (the changed subset) but NOT stamped above still carry # a stale semantic_hash from a prior run; clear it so detect_incremental re-queues # them instead of reading them as unchanged (#1948). @@ -589,7 +589,7 @@ _cleared = _dispatched - _stamped # files newly excluded since last run are dropped rather than masquerading as # deletions; untouched files' prior rows are still preserved (#1908). _scan = {f for fl in _corpus.values() for f in fl} -save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) # Update cumulative cost tracker input_tok = extract.get('input_tokens', 0) @@ -613,7 +613,7 @@ cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\" print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" +" "INPUT_PATH" rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null rm -f graphify-out/needs_update 2>/dev/null || true diff --git a/graphify/skill-kilo.md b/graphify/skill-kilo.md index 402fc962e7..05d3e9f4a0 100644 --- a/graphify/skill-kilo.md +++ b/graphify/skill-kilo.md @@ -108,15 +108,15 @@ If the import succeeds, print nothing and move straight to Step 2. ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from graphify.detect import detect from pathlib import Path -result = detect(Path('INPUT_PATH')) +result = detect(Path(sys.argv[1])) # Write the sidecar from Python, not a shell redirect, so the same block renders # on PowerShell hosts without console-encoding drift (#2528). Path('graphify-out/.graphify_detect.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") print(f'Detected {result[\"total_files\"]} files') -" +" "INPUT_PATH" ``` Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: @@ -183,13 +183,13 @@ for f in detect.get('files', {}).get('code', []): code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) if code_files: - result = extract(code_files, cache_root=Path('INPUT_PATH')) + result = extract(code_files, cache_root=Path(sys.argv[1])) Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') else: Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") print('No code files - skipping AST extraction') -" +" "INPUT_PATH" ``` #### Part B - Semantic extraction (parallel subagents) @@ -204,7 +204,7 @@ Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],' " ``` -**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** +**MANDATORY: You MUST use the subagent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the subagent tool you are doing this wrong.** Before dispatching subagents, print a timing estimate: - Load `total_words` and file counts from `graphify-out/.graphify_detect.json` @@ -220,7 +220,7 @@ SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from graphify.cache import check_semantic_cache from pathlib import Path @@ -230,7 +230,7 @@ detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encodin # every source file (#1392). Video is transcribed to a document in Step 2.5 first. all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root=sys.argv[1], prompt_file=sys.argv[2]) # Always (re)write the cache file: write hits, else DELETE any leftover from a prior # run so Part C never merges a stale .graphify_cached.json (#1392). @@ -240,7 +240,7 @@ else: Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" +" "INPUT_PATH" "SPEC_PATH" ``` Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. @@ -285,7 +285,7 @@ Wait for all subagents. For each result: If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: +Merge all chunk files into `.graphify_semantic_new.json`. **After each subagent call completes, read the real token counts from the subagent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: ```bash $(cat graphify-out/.graphify_python) -c " import json, glob @@ -312,15 +312,15 @@ print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens' Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root=sys.argv[1], allowed_source_files=uncached, prompt_file=sys.argv[2]) print(f'Cached {saved} files') -" +" "INPUT_PATH" "SPEC_PATH" ``` Merge cached + new results into `graphify-out/.graphify_semantic.json`: @@ -409,7 +409,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -434,7 +434,7 @@ if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') print('If this shrink is intentional (you deleted files), re-run a full build with --force.') raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, sys.argv[1], suggested_questions=questions) Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") analysis = { 'communities': {str(k): v for k, v in communities.items()}, @@ -445,7 +445,7 @@ analysis = { } Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" +" "INPUT_PATH" ``` If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. @@ -458,12 +458,12 @@ A non-destructive diagnostic on the extraction, before labeling. It surfaces edg ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from pathlib import Path from graphify.diagnostics import diagnose_extraction, format_diagnostic_report extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') +summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root=sys.argv[1]) print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( ('dangling_endpoint_edges', 'dangling-endpoint edges'), @@ -473,7 +473,7 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0)] print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') -" +" "INPUT_PATH" ``` Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules). @@ -499,7 +499,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} @@ -510,7 +510,7 @@ labels = LABELS_DICT # Regenerate questions with real community labels (labels affect question phrasing) questions = suggest_questions(G, communities, labels) -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, sys.argv[1], suggested_questions=questions) # Re-export so graph.json nodes carry the curated community_name (#2490). # Same extraction as Step 4, so the #479 shrink-guard passes on node count; # if it still refuses, surface the guard message - do not force past it. @@ -522,7 +522,7 @@ if not wrote: Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") print('Report updated with community labels') -" +" "INPUT_PATH" ``` Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). @@ -557,7 +557,7 @@ These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, ` ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from pathlib import Path from datetime import datetime, timezone from graphify.detect import save_manifest @@ -580,7 +580,7 @@ extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encod # types are gated on output. from graphify.cli import _stamped_manifest_files _corpus = detect.get('all_files') or detect['files'] -_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) +_manifest_files = _stamped_manifest_files(_corpus, extract, Path(sys.argv[1])) # Files dispatched this run (the changed subset) but NOT stamped above still carry # a stale semantic_hash from a prior run; clear it so detect_incremental re-queues # them instead of reading them as unchanged (#1948). @@ -592,7 +592,7 @@ _cleared = _dispatched - _stamped # files newly excluded since last run are dropped rather than masquerading as # deletions; untouched files' prior rows are still preserved (#1908). _scan = {f for fl in _corpus.values() for f in fl} -save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) # Update cumulative cost tracker input_tok = extract.get('input_tokens', 0) @@ -616,7 +616,7 @@ cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\" print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" +" "INPUT_PATH" rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null rm -f graphify-out/needs_update 2>/dev/null || true diff --git a/graphify/skill-kiro.md b/graphify/skill-kiro.md index 56c5c75fc2..7e7c797563 100644 --- a/graphify/skill-kiro.md +++ b/graphify/skill-kiro.md @@ -108,15 +108,15 @@ If the import succeeds, print nothing and move straight to Step 2. ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from graphify.detect import detect from pathlib import Path -result = detect(Path('INPUT_PATH')) +result = detect(Path(sys.argv[1])) # Write the sidecar from Python, not a shell redirect, so the same block renders # on PowerShell hosts without console-encoding drift (#2528). Path('graphify-out/.graphify_detect.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") print(f'Detected {result[\"total_files\"]} files') -" +" "INPUT_PATH" ``` Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: @@ -183,13 +183,13 @@ for f in detect.get('files', {}).get('code', []): code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) if code_files: - result = extract(code_files, cache_root=Path('INPUT_PATH')) + result = extract(code_files, cache_root=Path(sys.argv[1])) Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') else: Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") print('No code files - skipping AST extraction') -" +" "INPUT_PATH" ``` #### Part B - Semantic extraction (parallel subagents) @@ -204,7 +204,7 @@ Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],' " ``` -**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** +**MANDATORY: You MUST use the subagent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the subagent tool you are doing this wrong.** Before dispatching subagents, print a timing estimate: - Load `total_words` and file counts from `graphify-out/.graphify_detect.json` @@ -220,7 +220,7 @@ SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from graphify.cache import check_semantic_cache from pathlib import Path @@ -230,7 +230,7 @@ detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encodin # every source file (#1392). Video is transcribed to a document in Step 2.5 first. all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root=sys.argv[1], prompt_file=sys.argv[2]) # Always (re)write the cache file: write hits, else DELETE any leftover from a prior # run so Part C never merges a stale .graphify_cached.json (#1392). @@ -240,7 +240,7 @@ else: Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" +" "INPUT_PATH" "SPEC_PATH" ``` Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. @@ -285,7 +285,7 @@ Wait for all subagents. For each result: If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: +Merge all chunk files into `.graphify_semantic_new.json`. **After each subagent call completes, read the real token counts from the subagent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: ```bash $(cat graphify-out/.graphify_python) -c " import json, glob @@ -312,15 +312,15 @@ print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens' Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root=sys.argv[1], allowed_source_files=uncached, prompt_file=sys.argv[2]) print(f'Cached {saved} files') -" +" "INPUT_PATH" "SPEC_PATH" ``` Merge cached + new results into `graphify-out/.graphify_semantic.json`: @@ -409,7 +409,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -434,7 +434,7 @@ if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') print('If this shrink is intentional (you deleted files), re-run a full build with --force.') raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, sys.argv[1], suggested_questions=questions) Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") analysis = { 'communities': {str(k): v for k, v in communities.items()}, @@ -445,7 +445,7 @@ analysis = { } Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" +" "INPUT_PATH" ``` If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. @@ -458,12 +458,12 @@ A non-destructive diagnostic on the extraction, before labeling. It surfaces edg ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from pathlib import Path from graphify.diagnostics import diagnose_extraction, format_diagnostic_report extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') +summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root=sys.argv[1]) print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( ('dangling_endpoint_edges', 'dangling-endpoint edges'), @@ -473,7 +473,7 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0)] print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') -" +" "INPUT_PATH" ``` Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules). @@ -499,7 +499,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} @@ -510,7 +510,7 @@ labels = LABELS_DICT # Regenerate questions with real community labels (labels affect question phrasing) questions = suggest_questions(G, communities, labels) -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, sys.argv[1], suggested_questions=questions) # Re-export so graph.json nodes carry the curated community_name (#2490). # Same extraction as Step 4, so the #479 shrink-guard passes on node count; # if it still refuses, surface the guard message - do not force past it. @@ -522,7 +522,7 @@ if not wrote: Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") print('Report updated with community labels') -" +" "INPUT_PATH" ``` Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). @@ -557,7 +557,7 @@ These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, ` ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from pathlib import Path from datetime import datetime, timezone from graphify.detect import save_manifest @@ -580,7 +580,7 @@ extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encod # types are gated on output. from graphify.cli import _stamped_manifest_files _corpus = detect.get('all_files') or detect['files'] -_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) +_manifest_files = _stamped_manifest_files(_corpus, extract, Path(sys.argv[1])) # Files dispatched this run (the changed subset) but NOT stamped above still carry # a stale semantic_hash from a prior run; clear it so detect_incremental re-queues # them instead of reading them as unchanged (#1948). @@ -592,7 +592,7 @@ _cleared = _dispatched - _stamped # files newly excluded since last run are dropped rather than masquerading as # deletions; untouched files' prior rows are still preserved (#1908). _scan = {f for fl in _corpus.values() for f in fl} -save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) # Update cumulative cost tracker input_tok = extract.get('input_tokens', 0) @@ -616,7 +616,7 @@ cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\" print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" +" "INPUT_PATH" rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null rm -f graphify-out/needs_update 2>/dev/null || true diff --git a/graphify/skill-opencode.md b/graphify/skill-opencode.md index f8e24e6272..313958ae2d 100644 --- a/graphify/skill-opencode.md +++ b/graphify/skill-opencode.md @@ -108,15 +108,15 @@ If the import succeeds, print nothing and move straight to Step 2. ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from graphify.detect import detect from pathlib import Path -result = detect(Path('INPUT_PATH')) +result = detect(Path(sys.argv[1])) # Write the sidecar from Python, not a shell redirect, so the same block renders # on PowerShell hosts without console-encoding drift (#2528). Path('graphify-out/.graphify_detect.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") print(f'Detected {result[\"total_files\"]} files') -" +" "INPUT_PATH" ``` Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: @@ -183,13 +183,13 @@ for f in detect.get('files', {}).get('code', []): code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) if code_files: - result = extract(code_files, cache_root=Path('INPUT_PATH')) + result = extract(code_files, cache_root=Path(sys.argv[1])) Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') else: Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") print('No code files - skipping AST extraction') -" +" "INPUT_PATH" ``` #### Part B - Semantic extraction (parallel subagents) @@ -204,7 +204,7 @@ Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],' " ``` -**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** +**MANDATORY: You MUST use the subagent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the subagent tool you are doing this wrong.** Before dispatching subagents, print a timing estimate: - Load `total_words` and file counts from `graphify-out/.graphify_detect.json` @@ -220,7 +220,7 @@ SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from graphify.cache import check_semantic_cache from pathlib import Path @@ -230,7 +230,7 @@ detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encodin # every source file (#1392). Video is transcribed to a document in Step 2.5 first. all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root=sys.argv[1], prompt_file=sys.argv[2]) # Always (re)write the cache file: write hits, else DELETE any leftover from a prior # run so Part C never merges a stale .graphify_cached.json (#1392). @@ -240,7 +240,7 @@ else: Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" +" "INPUT_PATH" "SPEC_PATH" ``` Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. @@ -277,7 +277,7 @@ Wait for all subagents. For each result: If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: +Merge all chunk files into `.graphify_semantic_new.json`. **After each subagent call completes, read the real token counts from the subagent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: ```bash $(cat graphify-out/.graphify_python) -c " import json, glob @@ -304,15 +304,15 @@ print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens' Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root=sys.argv[1], allowed_source_files=uncached, prompt_file=sys.argv[2]) print(f'Cached {saved} files') -" +" "INPUT_PATH" "SPEC_PATH" ``` Merge cached + new results into `graphify-out/.graphify_semantic.json`: @@ -401,7 +401,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -426,7 +426,7 @@ if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') print('If this shrink is intentional (you deleted files), re-run a full build with --force.') raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, sys.argv[1], suggested_questions=questions) Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") analysis = { 'communities': {str(k): v for k, v in communities.items()}, @@ -437,7 +437,7 @@ analysis = { } Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" +" "INPUT_PATH" ``` If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. @@ -450,12 +450,12 @@ A non-destructive diagnostic on the extraction, before labeling. It surfaces edg ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from pathlib import Path from graphify.diagnostics import diagnose_extraction, format_diagnostic_report extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') +summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root=sys.argv[1]) print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( ('dangling_endpoint_edges', 'dangling-endpoint edges'), @@ -465,7 +465,7 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0)] print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') -" +" "INPUT_PATH" ``` Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules). @@ -491,7 +491,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} @@ -502,7 +502,7 @@ labels = LABELS_DICT # Regenerate questions with real community labels (labels affect question phrasing) questions = suggest_questions(G, communities, labels) -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, sys.argv[1], suggested_questions=questions) # Re-export so graph.json nodes carry the curated community_name (#2490). # Same extraction as Step 4, so the #479 shrink-guard passes on node count; # if it still refuses, surface the guard message - do not force past it. @@ -514,7 +514,7 @@ if not wrote: Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") print('Report updated with community labels') -" +" "INPUT_PATH" ``` Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). @@ -549,7 +549,7 @@ These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, ` ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from pathlib import Path from datetime import datetime, timezone from graphify.detect import save_manifest @@ -572,7 +572,7 @@ extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encod # types are gated on output. from graphify.cli import _stamped_manifest_files _corpus = detect.get('all_files') or detect['files'] -_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) +_manifest_files = _stamped_manifest_files(_corpus, extract, Path(sys.argv[1])) # Files dispatched this run (the changed subset) but NOT stamped above still carry # a stale semantic_hash from a prior run; clear it so detect_incremental re-queues # them instead of reading them as unchanged (#1948). @@ -584,7 +584,7 @@ _cleared = _dispatched - _stamped # files newly excluded since last run are dropped rather than masquerading as # deletions; untouched files' prior rows are still preserved (#1908). _scan = {f for fl in _corpus.values() for f in fl} -save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) # Update cumulative cost tracker input_tok = extract.get('input_tokens', 0) @@ -608,7 +608,7 @@ cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\" print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" +" "INPUT_PATH" rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null rm -f graphify-out/needs_update 2>/dev/null || true diff --git a/graphify/skill-pi.md b/graphify/skill-pi.md index 56c5c75fc2..7e7c797563 100644 --- a/graphify/skill-pi.md +++ b/graphify/skill-pi.md @@ -108,15 +108,15 @@ If the import succeeds, print nothing and move straight to Step 2. ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from graphify.detect import detect from pathlib import Path -result = detect(Path('INPUT_PATH')) +result = detect(Path(sys.argv[1])) # Write the sidecar from Python, not a shell redirect, so the same block renders # on PowerShell hosts without console-encoding drift (#2528). Path('graphify-out/.graphify_detect.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") print(f'Detected {result[\"total_files\"]} files') -" +" "INPUT_PATH" ``` Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: @@ -183,13 +183,13 @@ for f in detect.get('files', {}).get('code', []): code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) if code_files: - result = extract(code_files, cache_root=Path('INPUT_PATH')) + result = extract(code_files, cache_root=Path(sys.argv[1])) Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') else: Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") print('No code files - skipping AST extraction') -" +" "INPUT_PATH" ``` #### Part B - Semantic extraction (parallel subagents) @@ -204,7 +204,7 @@ Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],' " ``` -**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** +**MANDATORY: You MUST use the subagent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the subagent tool you are doing this wrong.** Before dispatching subagents, print a timing estimate: - Load `total_words` and file counts from `graphify-out/.graphify_detect.json` @@ -220,7 +220,7 @@ SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from graphify.cache import check_semantic_cache from pathlib import Path @@ -230,7 +230,7 @@ detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encodin # every source file (#1392). Video is transcribed to a document in Step 2.5 first. all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root=sys.argv[1], prompt_file=sys.argv[2]) # Always (re)write the cache file: write hits, else DELETE any leftover from a prior # run so Part C never merges a stale .graphify_cached.json (#1392). @@ -240,7 +240,7 @@ else: Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" +" "INPUT_PATH" "SPEC_PATH" ``` Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. @@ -285,7 +285,7 @@ Wait for all subagents. For each result: If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: +Merge all chunk files into `.graphify_semantic_new.json`. **After each subagent call completes, read the real token counts from the subagent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: ```bash $(cat graphify-out/.graphify_python) -c " import json, glob @@ -312,15 +312,15 @@ print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens' Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root=sys.argv[1], allowed_source_files=uncached, prompt_file=sys.argv[2]) print(f'Cached {saved} files') -" +" "INPUT_PATH" "SPEC_PATH" ``` Merge cached + new results into `graphify-out/.graphify_semantic.json`: @@ -409,7 +409,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -434,7 +434,7 @@ if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') print('If this shrink is intentional (you deleted files), re-run a full build with --force.') raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, sys.argv[1], suggested_questions=questions) Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") analysis = { 'communities': {str(k): v for k, v in communities.items()}, @@ -445,7 +445,7 @@ analysis = { } Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" +" "INPUT_PATH" ``` If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. @@ -458,12 +458,12 @@ A non-destructive diagnostic on the extraction, before labeling. It surfaces edg ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from pathlib import Path from graphify.diagnostics import diagnose_extraction, format_diagnostic_report extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') +summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root=sys.argv[1]) print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( ('dangling_endpoint_edges', 'dangling-endpoint edges'), @@ -473,7 +473,7 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0)] print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') -" +" "INPUT_PATH" ``` Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules). @@ -499,7 +499,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} @@ -510,7 +510,7 @@ labels = LABELS_DICT # Regenerate questions with real community labels (labels affect question phrasing) questions = suggest_questions(G, communities, labels) -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, sys.argv[1], suggested_questions=questions) # Re-export so graph.json nodes carry the curated community_name (#2490). # Same extraction as Step 4, so the #479 shrink-guard passes on node count; # if it still refuses, surface the guard message - do not force past it. @@ -522,7 +522,7 @@ if not wrote: Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") print('Report updated with community labels') -" +" "INPUT_PATH" ``` Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). @@ -557,7 +557,7 @@ These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, ` ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from pathlib import Path from datetime import datetime, timezone from graphify.detect import save_manifest @@ -580,7 +580,7 @@ extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encod # types are gated on output. from graphify.cli import _stamped_manifest_files _corpus = detect.get('all_files') or detect['files'] -_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) +_manifest_files = _stamped_manifest_files(_corpus, extract, Path(sys.argv[1])) # Files dispatched this run (the changed subset) but NOT stamped above still carry # a stale semantic_hash from a prior run; clear it so detect_incremental re-queues # them instead of reading them as unchanged (#1948). @@ -592,7 +592,7 @@ _cleared = _dispatched - _stamped # files newly excluded since last run are dropped rather than masquerading as # deletions; untouched files' prior rows are still preserved (#1908). _scan = {f for fl in _corpus.values() for f in fl} -save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) # Update cumulative cost tracker input_tok = extract.get('input_tokens', 0) @@ -616,7 +616,7 @@ cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\" print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" +" "INPUT_PATH" rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null rm -f graphify-out/needs_update 2>/dev/null || true diff --git a/graphify/skill-trae.md b/graphify/skill-trae.md index 55f8c2caca..2b73e78216 100644 --- a/graphify/skill-trae.md +++ b/graphify/skill-trae.md @@ -108,15 +108,15 @@ If the import succeeds, print nothing and move straight to Step 2. ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from graphify.detect import detect from pathlib import Path -result = detect(Path('INPUT_PATH')) +result = detect(Path(sys.argv[1])) # Write the sidecar from Python, not a shell redirect, so the same block renders # on PowerShell hosts without console-encoding drift (#2528). Path('graphify-out/.graphify_detect.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") print(f'Detected {result[\"total_files\"]} files') -" +" "INPUT_PATH" ``` Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: @@ -183,13 +183,13 @@ for f in detect.get('files', {}).get('code', []): code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) if code_files: - result = extract(code_files, cache_root=Path('INPUT_PATH')) + result = extract(code_files, cache_root=Path(sys.argv[1])) Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') else: Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") print('No code files - skipping AST extraction') -" +" "INPUT_PATH" ``` #### Part B - Semantic extraction (parallel subagents) @@ -204,7 +204,7 @@ Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],' " ``` -**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** +**MANDATORY: You MUST use the subagent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the subagent tool you are doing this wrong.** Before dispatching subagents, print a timing estimate: - Load `total_words` and file counts from `graphify-out/.graphify_detect.json` @@ -220,7 +220,7 @@ SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from graphify.cache import check_semantic_cache from pathlib import Path @@ -230,7 +230,7 @@ detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encodin # every source file (#1392). Video is transcribed to a document in Step 2.5 first. all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root=sys.argv[1], prompt_file=sys.argv[2]) # Always (re)write the cache file: write hits, else DELETE any leftover from a prior # run so Part C never merges a stale .graphify_cached.json (#1392). @@ -240,7 +240,7 @@ else: Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" +" "INPUT_PATH" "SPEC_PATH" ``` Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. @@ -283,7 +283,7 @@ Wait for all subagents. For each result: If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: +Merge all chunk files into `.graphify_semantic_new.json`. **After each subagent call completes, read the real token counts from the subagent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: ```bash $(cat graphify-out/.graphify_python) -c " import json, glob @@ -310,15 +310,15 @@ print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens' Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root=sys.argv[1], allowed_source_files=uncached, prompt_file=sys.argv[2]) print(f'Cached {saved} files') -" +" "INPUT_PATH" "SPEC_PATH" ``` Merge cached + new results into `graphify-out/.graphify_semantic.json`: @@ -407,7 +407,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -432,7 +432,7 @@ if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') print('If this shrink is intentional (you deleted files), re-run a full build with --force.') raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, sys.argv[1], suggested_questions=questions) Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") analysis = { 'communities': {str(k): v for k, v in communities.items()}, @@ -443,7 +443,7 @@ analysis = { } Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" +" "INPUT_PATH" ``` If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. @@ -456,12 +456,12 @@ A non-destructive diagnostic on the extraction, before labeling. It surfaces edg ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from pathlib import Path from graphify.diagnostics import diagnose_extraction, format_diagnostic_report extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') +summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root=sys.argv[1]) print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( ('dangling_endpoint_edges', 'dangling-endpoint edges'), @@ -471,7 +471,7 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0)] print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') -" +" "INPUT_PATH" ``` Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules). @@ -497,7 +497,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} @@ -508,7 +508,7 @@ labels = LABELS_DICT # Regenerate questions with real community labels (labels affect question phrasing) questions = suggest_questions(G, communities, labels) -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, sys.argv[1], suggested_questions=questions) # Re-export so graph.json nodes carry the curated community_name (#2490). # Same extraction as Step 4, so the #479 shrink-guard passes on node count; # if it still refuses, surface the guard message - do not force past it. @@ -520,7 +520,7 @@ if not wrote: Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") print('Report updated with community labels') -" +" "INPUT_PATH" ``` Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). @@ -555,7 +555,7 @@ These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, ` ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from pathlib import Path from datetime import datetime, timezone from graphify.detect import save_manifest @@ -578,7 +578,7 @@ extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encod # types are gated on output. from graphify.cli import _stamped_manifest_files _corpus = detect.get('all_files') or detect['files'] -_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) +_manifest_files = _stamped_manifest_files(_corpus, extract, Path(sys.argv[1])) # Files dispatched this run (the changed subset) but NOT stamped above still carry # a stale semantic_hash from a prior run; clear it so detect_incremental re-queues # them instead of reading them as unchanged (#1948). @@ -590,7 +590,7 @@ _cleared = _dispatched - _stamped # files newly excluded since last run are dropped rather than masquerading as # deletions; untouched files' prior rows are still preserved (#1908). _scan = {f for fl in _corpus.values() for f in fl} -save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) # Update cumulative cost tracker input_tok = extract.get('input_tokens', 0) @@ -614,7 +614,7 @@ cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\" print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" +" "INPUT_PATH" rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null rm -f graphify-out/needs_update 2>/dev/null || true diff --git a/graphify/skill-vscode.md b/graphify/skill-vscode.md index 9151a264b2..c187cd7d33 100644 --- a/graphify/skill-vscode.md +++ b/graphify/skill-vscode.md @@ -108,15 +108,15 @@ If the import succeeds, print nothing and move straight to Step 2. ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from graphify.detect import detect from pathlib import Path -result = detect(Path('INPUT_PATH')) +result = detect(Path(sys.argv[1])) # Write the sidecar from Python, not a shell redirect, so the same block renders # on PowerShell hosts without console-encoding drift (#2528). Path('graphify-out/.graphify_detect.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") print(f'Detected {result[\"total_files\"]} files') -" +" "INPUT_PATH" ``` Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: @@ -183,13 +183,13 @@ for f in detect.get('files', {}).get('code', []): code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) if code_files: - result = extract(code_files, cache_root=Path('INPUT_PATH')) + result = extract(code_files, cache_root=Path(sys.argv[1])) Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') else: Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") print('No code files - skipping AST extraction') -" +" "INPUT_PATH" ``` #### Part B - Semantic extraction (parallel subagents) @@ -204,7 +204,7 @@ Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],' " ``` -**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** +**MANDATORY: You MUST use the subagent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the subagent tool you are doing this wrong.** Before dispatching subagents, print a timing estimate: - Load `total_words` and file counts from `graphify-out/.graphify_detect.json` @@ -220,7 +220,7 @@ SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from graphify.cache import check_semantic_cache from pathlib import Path @@ -230,7 +230,7 @@ detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encodin # every source file (#1392). Video is transcribed to a document in Step 2.5 first. all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root=sys.argv[1], prompt_file=sys.argv[2]) # Always (re)write the cache file: write hits, else DELETE any leftover from a prior # run so Part C never merges a stale .graphify_cached.json (#1392). @@ -240,7 +240,7 @@ else: Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" +" "INPUT_PATH" "SPEC_PATH" ``` Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. @@ -281,7 +281,7 @@ Wait for all subagents. For each result: If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: +Merge all chunk files into `.graphify_semantic_new.json`. **After each subagent call completes, read the real token counts from the subagent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: ```bash $(cat graphify-out/.graphify_python) -c " import json, glob @@ -308,15 +308,15 @@ print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens' Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root=sys.argv[1], allowed_source_files=uncached, prompt_file=sys.argv[2]) print(f'Cached {saved} files') -" +" "INPUT_PATH" "SPEC_PATH" ``` Merge cached + new results into `graphify-out/.graphify_semantic.json`: @@ -405,7 +405,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -430,7 +430,7 @@ if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') print('If this shrink is intentional (you deleted files), re-run a full build with --force.') raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, sys.argv[1], suggested_questions=questions) Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") analysis = { 'communities': {str(k): v for k, v in communities.items()}, @@ -441,7 +441,7 @@ analysis = { } Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" +" "INPUT_PATH" ``` If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. @@ -454,12 +454,12 @@ A non-destructive diagnostic on the extraction, before labeling. It surfaces edg ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from pathlib import Path from graphify.diagnostics import diagnose_extraction, format_diagnostic_report extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') +summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root=sys.argv[1]) print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( ('dangling_endpoint_edges', 'dangling-endpoint edges'), @@ -469,7 +469,7 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0)] print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') -" +" "INPUT_PATH" ``` Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules). @@ -495,7 +495,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} @@ -506,7 +506,7 @@ labels = LABELS_DICT # Regenerate questions with real community labels (labels affect question phrasing) questions = suggest_questions(G, communities, labels) -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, sys.argv[1], suggested_questions=questions) # Re-export so graph.json nodes carry the curated community_name (#2490). # Same extraction as Step 4, so the #479 shrink-guard passes on node count; # if it still refuses, surface the guard message - do not force past it. @@ -518,7 +518,7 @@ if not wrote: Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") print('Report updated with community labels') -" +" "INPUT_PATH" ``` Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). @@ -553,7 +553,7 @@ These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, ` ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from pathlib import Path from datetime import datetime, timezone from graphify.detect import save_manifest @@ -576,7 +576,7 @@ extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encod # types are gated on output. from graphify.cli import _stamped_manifest_files _corpus = detect.get('all_files') or detect['files'] -_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) +_manifest_files = _stamped_manifest_files(_corpus, extract, Path(sys.argv[1])) # Files dispatched this run (the changed subset) but NOT stamped above still carry # a stale semantic_hash from a prior run; clear it so detect_incremental re-queues # them instead of reading them as unchanged (#1948). @@ -588,7 +588,7 @@ _cleared = _dispatched - _stamped # files newly excluded since last run are dropped rather than masquerading as # deletions; untouched files' prior rows are still preserved (#1908). _scan = {f for fl in _corpus.values() for f in fl} -save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) # Update cumulative cost tracker input_tok = extract.get('input_tokens', 0) @@ -612,7 +612,7 @@ cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\" print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" +" "INPUT_PATH" rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null rm -f graphify-out/needs_update 2>/dev/null || true diff --git a/graphify/skill-windows.md b/graphify/skill-windows.md index 81dc1b133c..9534fbd706 100644 --- a/graphify/skill-windows.md +++ b/graphify/skill-windows.md @@ -135,15 +135,15 @@ If the import succeeds, print nothing and move straight to Step 2. ```powershell @' -import json +import sys, json from graphify.detect import detect from pathlib import Path -result = detect(Path('INPUT_PATH')) +result = detect(Path(sys.argv[1])) # Write the sidecar from Python, not a shell redirect, so the same block renders # on PowerShell hosts without console-encoding drift (#2528). Path('graphify-out/.graphify_detect.json').write_text(json.dumps(result, ensure_ascii=False), encoding="utf-8") print(f'Detected {result["total_files"]} files') -'@ | & (Get-Content graphify-out\.graphify_python) - +'@ | & (Get-Content graphify-out\.graphify_python) - "INPUT_PATH" ``` Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: @@ -210,13 +210,13 @@ for f in detect.get('files', {}).get('code', []): code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) if code_files: - result = extract(code_files, cache_root=Path('INPUT_PATH')) + result = extract(code_files, cache_root=Path(sys.argv[1])) Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding="utf-8") print(f'AST: {len(result["nodes"])} nodes, {len(result["edges"])} edges') else: Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding="utf-8") print('No code files - skipping AST extraction') -'@ | & (Get-Content graphify-out\.graphify_python) - +'@ | & (Get-Content graphify-out\.graphify_python) - "INPUT_PATH" ``` #### Part B - Semantic extraction (parallel subagents) @@ -231,7 +231,7 @@ Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],' '@ | & (Get-Content graphify-out\.graphify_python) - ``` -**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** +**MANDATORY: You MUST use the subagent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the subagent tool you are doing this wrong.** Before dispatching subagents, print a timing estimate: - Load `total_words` and file counts from `graphify-out/.graphify_detect.json` @@ -247,7 +247,7 @@ SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` ```powershell @' -import json +import sys, json from graphify.cache import check_semantic_cache from pathlib import Path @@ -257,7 +257,7 @@ detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encodin # every source file (#1392). Video is transcribed to a document in Step 2.5 first. all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root=sys.argv[1], prompt_file=sys.argv[2]) # Always (re)write the cache file: write hits, else DELETE any leftover from a prior # run so Part C never merges a stale .graphify_cached.json (#1392). @@ -267,7 +267,7 @@ else: Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding="utf-8") print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -'@ | & (Get-Content graphify-out\.graphify_python) - +'@ | & (Get-Content graphify-out\.graphify_python) - "INPUT_PATH" "SPEC_PATH" ``` Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. @@ -312,7 +312,7 @@ Wait for all subagents. For each result: If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: +Merge all chunk files into `.graphify_semantic_new.json`. **After each subagent call completes, read the real token counts from the subagent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: ```powershell @' import json, glob @@ -339,15 +339,15 @@ print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens' Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): ```powershell @' -import json +import sys, json from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding="utf-8")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding="utf-8").splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root=sys.argv[1], allowed_source_files=uncached, prompt_file=sys.argv[2]) print(f'Cached {saved} files') -'@ | & (Get-Content graphify-out\.graphify_python) - +'@ | & (Get-Content graphify-out\.graphify_python) - "INPUT_PATH" "SPEC_PATH" ``` Merge cached + new results into `graphify-out/.graphify_semantic.json`: @@ -436,7 +436,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -461,7 +461,7 @@ if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') print('If this shrink is intentional (you deleted files), re-run a full build with --force.') raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, sys.argv[1], suggested_questions=questions) Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding="utf-8") analysis = { 'communities': {str(k): v for k, v in communities.items()}, @@ -472,7 +472,7 @@ analysis = { } Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding="utf-8") print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -'@ | & (Get-Content graphify-out\.graphify_python) - +'@ | & (Get-Content graphify-out\.graphify_python) - "INPUT_PATH" ``` If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. @@ -485,12 +485,12 @@ A non-destructive diagnostic on the extraction, before labeling. It surfaces edg ```powershell @' -import json +import sys, json from pathlib import Path from graphify.diagnostics import diagnose_extraction, format_diagnostic_report extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding="utf-8")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') +summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root=sys.argv[1]) print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( ('dangling_endpoint_edges', 'dangling-endpoint edges'), @@ -500,7 +500,7 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0)] print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') -'@ | & (Get-Content graphify-out\.graphify_python) - +'@ | & (Get-Content graphify-out\.graphify_python) - "INPUT_PATH" ``` Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules). @@ -526,7 +526,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding="utf-8")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} @@ -537,7 +537,7 @@ labels = LABELS_DICT # Regenerate questions with real community labels (labels affect question phrasing) questions = suggest_questions(G, communities, labels) -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, sys.argv[1], suggested_questions=questions) # Re-export so graph.json nodes carry the curated community_name (#2490). # Same extraction as Step 4, so the #479 shrink-guard passes on node count; # if it still refuses, surface the guard message - do not force past it. @@ -549,7 +549,7 @@ if not wrote: Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding="utf-8") Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding="utf-8") print('Report updated with community labels') -'@ | & (Get-Content graphify-out\.graphify_python) - +'@ | & (Get-Content graphify-out\.graphify_python) - "INPUT_PATH" ``` Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). @@ -584,7 +584,7 @@ These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, ` ```powershell @' -import json +import sys, json from pathlib import Path from datetime import datetime, timezone from graphify.detect import save_manifest @@ -607,7 +607,7 @@ extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encod # types are gated on output. from graphify.cli import _stamped_manifest_files _corpus = detect.get('all_files') or detect['files'] -_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) +_manifest_files = _stamped_manifest_files(_corpus, extract, Path(sys.argv[1])) # Files dispatched this run (the changed subset) but NOT stamped above still carry # a stale semantic_hash from a prior run; clear it so detect_incremental re-queues # them instead of reading them as unchanged (#1948). @@ -619,7 +619,7 @@ _cleared = _dispatched - _stamped # files newly excluded since last run are dropped rather than masquerading as # deletions; untouched files' prior rows are still preserved (#1908). _scan = {f for fl in _corpus.values() for f in fl} -save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) # Update cumulative cost tracker input_tok = extract.get('input_tokens', 0) @@ -643,7 +643,7 @@ cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding="u print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') print(f'All time: {cost["total_input_tokens"]:,} input, {cost["total_output_tokens"]:,} output ({len(cost["runs"])} runs)') -'@ | & (Get-Content graphify-out\.graphify_python) - +'@ | & (Get-Content graphify-out\.graphify_python) - "INPUT_PATH" Remove-Item -Force -ErrorAction SilentlyContinue graphify-out\.graphify_detect.json, graphify-out\.graphify_extract.json, graphify-out\.graphify_ast.json, graphify-out\.graphify_semantic.json, graphify-out\.graphify_analysis.json Get-ChildItem graphify-out -Filter '.graphify_chunk_*.json' -File -ErrorAction SilentlyContinue | Remove-Item -Force Remove-Item -Force -ErrorAction SilentlyContinue graphify-out\needs_update diff --git a/graphify/skill.md b/graphify/skill.md index 56c5c75fc2..7e7c797563 100644 --- a/graphify/skill.md +++ b/graphify/skill.md @@ -108,15 +108,15 @@ If the import succeeds, print nothing and move straight to Step 2. ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from graphify.detect import detect from pathlib import Path -result = detect(Path('INPUT_PATH')) +result = detect(Path(sys.argv[1])) # Write the sidecar from Python, not a shell redirect, so the same block renders # on PowerShell hosts without console-encoding drift (#2528). Path('graphify-out/.graphify_detect.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") print(f'Detected {result[\"total_files\"]} files') -" +" "INPUT_PATH" ``` Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: @@ -183,13 +183,13 @@ for f in detect.get('files', {}).get('code', []): code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) if code_files: - result = extract(code_files, cache_root=Path('INPUT_PATH')) + result = extract(code_files, cache_root=Path(sys.argv[1])) Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') else: Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") print('No code files - skipping AST extraction') -" +" "INPUT_PATH" ``` #### Part B - Semantic extraction (parallel subagents) @@ -204,7 +204,7 @@ Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],' " ``` -**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** +**MANDATORY: You MUST use the subagent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the subagent tool you are doing this wrong.** Before dispatching subagents, print a timing estimate: - Load `total_words` and file counts from `graphify-out/.graphify_detect.json` @@ -220,7 +220,7 @@ SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from graphify.cache import check_semantic_cache from pathlib import Path @@ -230,7 +230,7 @@ detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encodin # every source file (#1392). Video is transcribed to a document in Step 2.5 first. all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root=sys.argv[1], prompt_file=sys.argv[2]) # Always (re)write the cache file: write hits, else DELETE any leftover from a prior # run so Part C never merges a stale .graphify_cached.json (#1392). @@ -240,7 +240,7 @@ else: Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" +" "INPUT_PATH" "SPEC_PATH" ``` Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. @@ -285,7 +285,7 @@ Wait for all subagents. For each result: If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: +Merge all chunk files into `.graphify_semantic_new.json`. **After each subagent call completes, read the real token counts from the subagent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: ```bash $(cat graphify-out/.graphify_python) -c " import json, glob @@ -312,15 +312,15 @@ print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens' Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root=sys.argv[1], allowed_source_files=uncached, prompt_file=sys.argv[2]) print(f'Cached {saved} files') -" +" "INPUT_PATH" "SPEC_PATH" ``` Merge cached + new results into `graphify-out/.graphify_semantic.json`: @@ -409,7 +409,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -434,7 +434,7 @@ if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') print('If this shrink is intentional (you deleted files), re-run a full build with --force.') raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, sys.argv[1], suggested_questions=questions) Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") analysis = { 'communities': {str(k): v for k, v in communities.items()}, @@ -445,7 +445,7 @@ analysis = { } Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" +" "INPUT_PATH" ``` If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. @@ -458,12 +458,12 @@ A non-destructive diagnostic on the extraction, before labeling. It surfaces edg ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from pathlib import Path from graphify.diagnostics import diagnose_extraction, format_diagnostic_report extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') +summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root=sys.argv[1]) print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( ('dangling_endpoint_edges', 'dangling-endpoint edges'), @@ -473,7 +473,7 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0)] print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') -" +" "INPUT_PATH" ``` Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules). @@ -499,7 +499,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} @@ -510,7 +510,7 @@ labels = LABELS_DICT # Regenerate questions with real community labels (labels affect question phrasing) questions = suggest_questions(G, communities, labels) -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, sys.argv[1], suggested_questions=questions) # Re-export so graph.json nodes carry the curated community_name (#2490). # Same extraction as Step 4, so the #479 shrink-guard passes on node count; # if it still refuses, surface the guard message - do not force past it. @@ -522,7 +522,7 @@ if not wrote: Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") print('Report updated with community labels') -" +" "INPUT_PATH" ``` Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). @@ -557,7 +557,7 @@ These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, ` ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from pathlib import Path from datetime import datetime, timezone from graphify.detect import save_manifest @@ -580,7 +580,7 @@ extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encod # types are gated on output. from graphify.cli import _stamped_manifest_files _corpus = detect.get('all_files') or detect['files'] -_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) +_manifest_files = _stamped_manifest_files(_corpus, extract, Path(sys.argv[1])) # Files dispatched this run (the changed subset) but NOT stamped above still carry # a stale semantic_hash from a prior run; clear it so detect_incremental re-queues # them instead of reading them as unchanged (#1948). @@ -592,7 +592,7 @@ _cleared = _dispatched - _stamped # files newly excluded since last run are dropped rather than masquerading as # deletions; untouched files' prior rows are still preserved (#1908). _scan = {f for fl in _corpus.values() for f in fl} -save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) # Update cumulative cost tracker input_tok = extract.get('input_tokens', 0) @@ -616,7 +616,7 @@ cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\" print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" +" "INPUT_PATH" rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null rm -f graphify-out/needs_update 2>/dev/null || true diff --git a/graphify/skills/agents/references/transcribe.md b/graphify/skills/agents/references/transcribe.md index b967f83799..bbe8b93e45 100644 --- a/graphify/skills/agents/references/transcribe.md +++ b/graphify/skills/agents/references/transcribe.md @@ -8,13 +8,11 @@ Skip this step entirely if `detect` returned zero `video` files. Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. -**Strategy:** Read the god nodes from `graphify-out/.graphify_detect.json` (or the analysis file if it exists from a previous run). You are already a language model — write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. - -**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` +**Strategy:** If `graphify-out/.graphify_analysis.json` exists from a previous run, read its top god-node labels and write a one-sentence domain hint from them. Otherwise use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` Pass the chosen prompt to Whisper as its initial prompt. No separate API call needed. **Step 1 - Write the Whisper prompt yourself.** -Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: +Read the top god-node labels from previous analysis when available, then compose a short domain hint sentence. With no previous analysis, use the generic fallback above. - Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` - Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` diff --git a/graphify/skills/agents/references/update.md b/graphify/skills/agents/references/update.md index bd8e659d3c..058ca8e6d2 100644 --- a/graphify/skills/agents/references/update.md +++ b/graphify/skills/agents/references/update.md @@ -12,7 +12,7 @@ import sys, json from graphify.detect import detect_incremental, save_manifest from pathlib import Path -result = detect_incremental(Path('INPUT_PATH')) +result = detect_incremental(Path(sys.argv[1])) new_total = result.get('new_total', 0) print(json.dumps(result, indent=2, ensure_ascii=False)) Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") @@ -24,7 +24,7 @@ if deleted: print(f'{len(deleted)} deleted file(s) to prune.') if new_total > 0: print(f'{new_total} new/changed file(s) to re-extract.') -" +" "INPUT_PATH" ``` Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: @@ -84,7 +84,7 @@ Then: ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from pathlib import Path from graphify.build import build_merge from graphify.detect import save_manifest @@ -113,7 +113,7 @@ G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, - root='INPUT_PATH', + root=sys.argv[1], directed=IS_DIRECTED, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -150,7 +150,7 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) # is lost forever (#2015). Mirrors the library extract path # (cli._stamped_manifest_files + clear_semantic + scan_corpus). from graphify.cli import _stamped_manifest_files -_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) +_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path(sys.argv[1])) # Changed semantic files dispatched this run but NOT stamped had their chunk fail # or be omitted; clear any stale semantic_hash so they are re-queued (#1948). _sem_types = ('document', 'paper', 'image') @@ -160,9 +160,9 @@ _cleared = _dispatched - _stamped # scan_corpus = the RAW full corpus so in-root files newly excluded since last run # are dropped rather than masquerading as deletions; untouched rows preserved (#1908). _scan = {f for fl in incremental['files'].values() for f in fl} -save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) print('[graphify update] Manifest saved.') -" +" "INPUT_PATH" ``` Then run Steps 4–8 on the merged graph as normal. diff --git a/graphify/skills/amp/references/transcribe.md b/graphify/skills/amp/references/transcribe.md index b967f83799..bbe8b93e45 100644 --- a/graphify/skills/amp/references/transcribe.md +++ b/graphify/skills/amp/references/transcribe.md @@ -8,13 +8,11 @@ Skip this step entirely if `detect` returned zero `video` files. Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. -**Strategy:** Read the god nodes from `graphify-out/.graphify_detect.json` (or the analysis file if it exists from a previous run). You are already a language model — write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. - -**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` +**Strategy:** If `graphify-out/.graphify_analysis.json` exists from a previous run, read its top god-node labels and write a one-sentence domain hint from them. Otherwise use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` Pass the chosen prompt to Whisper as its initial prompt. No separate API call needed. **Step 1 - Write the Whisper prompt yourself.** -Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: +Read the top god-node labels from previous analysis when available, then compose a short domain hint sentence. With no previous analysis, use the generic fallback above. - Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` - Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` diff --git a/graphify/skills/amp/references/update.md b/graphify/skills/amp/references/update.md index bd8e659d3c..058ca8e6d2 100644 --- a/graphify/skills/amp/references/update.md +++ b/graphify/skills/amp/references/update.md @@ -12,7 +12,7 @@ import sys, json from graphify.detect import detect_incremental, save_manifest from pathlib import Path -result = detect_incremental(Path('INPUT_PATH')) +result = detect_incremental(Path(sys.argv[1])) new_total = result.get('new_total', 0) print(json.dumps(result, indent=2, ensure_ascii=False)) Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") @@ -24,7 +24,7 @@ if deleted: print(f'{len(deleted)} deleted file(s) to prune.') if new_total > 0: print(f'{new_total} new/changed file(s) to re-extract.') -" +" "INPUT_PATH" ``` Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: @@ -84,7 +84,7 @@ Then: ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from pathlib import Path from graphify.build import build_merge from graphify.detect import save_manifest @@ -113,7 +113,7 @@ G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, - root='INPUT_PATH', + root=sys.argv[1], directed=IS_DIRECTED, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -150,7 +150,7 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) # is lost forever (#2015). Mirrors the library extract path # (cli._stamped_manifest_files + clear_semantic + scan_corpus). from graphify.cli import _stamped_manifest_files -_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) +_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path(sys.argv[1])) # Changed semantic files dispatched this run but NOT stamped had their chunk fail # or be omitted; clear any stale semantic_hash so they are re-queued (#1948). _sem_types = ('document', 'paper', 'image') @@ -160,9 +160,9 @@ _cleared = _dispatched - _stamped # scan_corpus = the RAW full corpus so in-root files newly excluded since last run # are dropped rather than masquerading as deletions; untouched rows preserved (#1908). _scan = {f for fl in incremental['files'].values() for f in fl} -save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) print('[graphify update] Manifest saved.') -" +" "INPUT_PATH" ``` Then run Steps 4–8 on the merged graph as normal. diff --git a/graphify/skills/claude/references/transcribe.md b/graphify/skills/claude/references/transcribe.md index b967f83799..bbe8b93e45 100644 --- a/graphify/skills/claude/references/transcribe.md +++ b/graphify/skills/claude/references/transcribe.md @@ -8,13 +8,11 @@ Skip this step entirely if `detect` returned zero `video` files. Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. -**Strategy:** Read the god nodes from `graphify-out/.graphify_detect.json` (or the analysis file if it exists from a previous run). You are already a language model — write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. - -**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` +**Strategy:** If `graphify-out/.graphify_analysis.json` exists from a previous run, read its top god-node labels and write a one-sentence domain hint from them. Otherwise use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` Pass the chosen prompt to Whisper as its initial prompt. No separate API call needed. **Step 1 - Write the Whisper prompt yourself.** -Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: +Read the top god-node labels from previous analysis when available, then compose a short domain hint sentence. With no previous analysis, use the generic fallback above. - Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` - Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` diff --git a/graphify/skills/claude/references/update.md b/graphify/skills/claude/references/update.md index bd8e659d3c..058ca8e6d2 100644 --- a/graphify/skills/claude/references/update.md +++ b/graphify/skills/claude/references/update.md @@ -12,7 +12,7 @@ import sys, json from graphify.detect import detect_incremental, save_manifest from pathlib import Path -result = detect_incremental(Path('INPUT_PATH')) +result = detect_incremental(Path(sys.argv[1])) new_total = result.get('new_total', 0) print(json.dumps(result, indent=2, ensure_ascii=False)) Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") @@ -24,7 +24,7 @@ if deleted: print(f'{len(deleted)} deleted file(s) to prune.') if new_total > 0: print(f'{new_total} new/changed file(s) to re-extract.') -" +" "INPUT_PATH" ``` Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: @@ -84,7 +84,7 @@ Then: ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from pathlib import Path from graphify.build import build_merge from graphify.detect import save_manifest @@ -113,7 +113,7 @@ G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, - root='INPUT_PATH', + root=sys.argv[1], directed=IS_DIRECTED, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -150,7 +150,7 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) # is lost forever (#2015). Mirrors the library extract path # (cli._stamped_manifest_files + clear_semantic + scan_corpus). from graphify.cli import _stamped_manifest_files -_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) +_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path(sys.argv[1])) # Changed semantic files dispatched this run but NOT stamped had their chunk fail # or be omitted; clear any stale semantic_hash so they are re-queued (#1948). _sem_types = ('document', 'paper', 'image') @@ -160,9 +160,9 @@ _cleared = _dispatched - _stamped # scan_corpus = the RAW full corpus so in-root files newly excluded since last run # are dropped rather than masquerading as deletions; untouched rows preserved (#1908). _scan = {f for fl in incremental['files'].values() for f in fl} -save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) print('[graphify update] Manifest saved.') -" +" "INPUT_PATH" ``` Then run Steps 4–8 on the merged graph as normal. diff --git a/graphify/skills/claw/references/transcribe.md b/graphify/skills/claw/references/transcribe.md index b967f83799..bbe8b93e45 100644 --- a/graphify/skills/claw/references/transcribe.md +++ b/graphify/skills/claw/references/transcribe.md @@ -8,13 +8,11 @@ Skip this step entirely if `detect` returned zero `video` files. Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. -**Strategy:** Read the god nodes from `graphify-out/.graphify_detect.json` (or the analysis file if it exists from a previous run). You are already a language model — write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. - -**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` +**Strategy:** If `graphify-out/.graphify_analysis.json` exists from a previous run, read its top god-node labels and write a one-sentence domain hint from them. Otherwise use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` Pass the chosen prompt to Whisper as its initial prompt. No separate API call needed. **Step 1 - Write the Whisper prompt yourself.** -Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: +Read the top god-node labels from previous analysis when available, then compose a short domain hint sentence. With no previous analysis, use the generic fallback above. - Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` - Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` diff --git a/graphify/skills/claw/references/update.md b/graphify/skills/claw/references/update.md index bd8e659d3c..058ca8e6d2 100644 --- a/graphify/skills/claw/references/update.md +++ b/graphify/skills/claw/references/update.md @@ -12,7 +12,7 @@ import sys, json from graphify.detect import detect_incremental, save_manifest from pathlib import Path -result = detect_incremental(Path('INPUT_PATH')) +result = detect_incremental(Path(sys.argv[1])) new_total = result.get('new_total', 0) print(json.dumps(result, indent=2, ensure_ascii=False)) Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") @@ -24,7 +24,7 @@ if deleted: print(f'{len(deleted)} deleted file(s) to prune.') if new_total > 0: print(f'{new_total} new/changed file(s) to re-extract.') -" +" "INPUT_PATH" ``` Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: @@ -84,7 +84,7 @@ Then: ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from pathlib import Path from graphify.build import build_merge from graphify.detect import save_manifest @@ -113,7 +113,7 @@ G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, - root='INPUT_PATH', + root=sys.argv[1], directed=IS_DIRECTED, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -150,7 +150,7 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) # is lost forever (#2015). Mirrors the library extract path # (cli._stamped_manifest_files + clear_semantic + scan_corpus). from graphify.cli import _stamped_manifest_files -_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) +_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path(sys.argv[1])) # Changed semantic files dispatched this run but NOT stamped had their chunk fail # or be omitted; clear any stale semantic_hash so they are re-queued (#1948). _sem_types = ('document', 'paper', 'image') @@ -160,9 +160,9 @@ _cleared = _dispatched - _stamped # scan_corpus = the RAW full corpus so in-root files newly excluded since last run # are dropped rather than masquerading as deletions; untouched rows preserved (#1908). _scan = {f for fl in incremental['files'].values() for f in fl} -save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) print('[graphify update] Manifest saved.') -" +" "INPUT_PATH" ``` Then run Steps 4–8 on the merged graph as normal. diff --git a/graphify/skills/codex/references/transcribe.md b/graphify/skills/codex/references/transcribe.md index b967f83799..bbe8b93e45 100644 --- a/graphify/skills/codex/references/transcribe.md +++ b/graphify/skills/codex/references/transcribe.md @@ -8,13 +8,11 @@ Skip this step entirely if `detect` returned zero `video` files. Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. -**Strategy:** Read the god nodes from `graphify-out/.graphify_detect.json` (or the analysis file if it exists from a previous run). You are already a language model — write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. - -**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` +**Strategy:** If `graphify-out/.graphify_analysis.json` exists from a previous run, read its top god-node labels and write a one-sentence domain hint from them. Otherwise use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` Pass the chosen prompt to Whisper as its initial prompt. No separate API call needed. **Step 1 - Write the Whisper prompt yourself.** -Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: +Read the top god-node labels from previous analysis when available, then compose a short domain hint sentence. With no previous analysis, use the generic fallback above. - Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` - Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` diff --git a/graphify/skills/codex/references/update.md b/graphify/skills/codex/references/update.md index bd8e659d3c..058ca8e6d2 100644 --- a/graphify/skills/codex/references/update.md +++ b/graphify/skills/codex/references/update.md @@ -12,7 +12,7 @@ import sys, json from graphify.detect import detect_incremental, save_manifest from pathlib import Path -result = detect_incremental(Path('INPUT_PATH')) +result = detect_incremental(Path(sys.argv[1])) new_total = result.get('new_total', 0) print(json.dumps(result, indent=2, ensure_ascii=False)) Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") @@ -24,7 +24,7 @@ if deleted: print(f'{len(deleted)} deleted file(s) to prune.') if new_total > 0: print(f'{new_total} new/changed file(s) to re-extract.') -" +" "INPUT_PATH" ``` Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: @@ -84,7 +84,7 @@ Then: ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from pathlib import Path from graphify.build import build_merge from graphify.detect import save_manifest @@ -113,7 +113,7 @@ G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, - root='INPUT_PATH', + root=sys.argv[1], directed=IS_DIRECTED, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -150,7 +150,7 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) # is lost forever (#2015). Mirrors the library extract path # (cli._stamped_manifest_files + clear_semantic + scan_corpus). from graphify.cli import _stamped_manifest_files -_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) +_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path(sys.argv[1])) # Changed semantic files dispatched this run but NOT stamped had their chunk fail # or be omitted; clear any stale semantic_hash so they are re-queued (#1948). _sem_types = ('document', 'paper', 'image') @@ -160,9 +160,9 @@ _cleared = _dispatched - _stamped # scan_corpus = the RAW full corpus so in-root files newly excluded since last run # are dropped rather than masquerading as deletions; untouched rows preserved (#1908). _scan = {f for fl in incremental['files'].values() for f in fl} -save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) print('[graphify update] Manifest saved.') -" +" "INPUT_PATH" ``` Then run Steps 4–8 on the merged graph as normal. diff --git a/graphify/skills/copilot/references/transcribe.md b/graphify/skills/copilot/references/transcribe.md index b967f83799..bbe8b93e45 100644 --- a/graphify/skills/copilot/references/transcribe.md +++ b/graphify/skills/copilot/references/transcribe.md @@ -8,13 +8,11 @@ Skip this step entirely if `detect` returned zero `video` files. Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. -**Strategy:** Read the god nodes from `graphify-out/.graphify_detect.json` (or the analysis file if it exists from a previous run). You are already a language model — write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. - -**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` +**Strategy:** If `graphify-out/.graphify_analysis.json` exists from a previous run, read its top god-node labels and write a one-sentence domain hint from them. Otherwise use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` Pass the chosen prompt to Whisper as its initial prompt. No separate API call needed. **Step 1 - Write the Whisper prompt yourself.** -Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: +Read the top god-node labels from previous analysis when available, then compose a short domain hint sentence. With no previous analysis, use the generic fallback above. - Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` - Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` diff --git a/graphify/skills/copilot/references/update.md b/graphify/skills/copilot/references/update.md index bd8e659d3c..058ca8e6d2 100644 --- a/graphify/skills/copilot/references/update.md +++ b/graphify/skills/copilot/references/update.md @@ -12,7 +12,7 @@ import sys, json from graphify.detect import detect_incremental, save_manifest from pathlib import Path -result = detect_incremental(Path('INPUT_PATH')) +result = detect_incremental(Path(sys.argv[1])) new_total = result.get('new_total', 0) print(json.dumps(result, indent=2, ensure_ascii=False)) Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") @@ -24,7 +24,7 @@ if deleted: print(f'{len(deleted)} deleted file(s) to prune.') if new_total > 0: print(f'{new_total} new/changed file(s) to re-extract.') -" +" "INPUT_PATH" ``` Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: @@ -84,7 +84,7 @@ Then: ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from pathlib import Path from graphify.build import build_merge from graphify.detect import save_manifest @@ -113,7 +113,7 @@ G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, - root='INPUT_PATH', + root=sys.argv[1], directed=IS_DIRECTED, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -150,7 +150,7 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) # is lost forever (#2015). Mirrors the library extract path # (cli._stamped_manifest_files + clear_semantic + scan_corpus). from graphify.cli import _stamped_manifest_files -_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) +_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path(sys.argv[1])) # Changed semantic files dispatched this run but NOT stamped had their chunk fail # or be omitted; clear any stale semantic_hash so they are re-queued (#1948). _sem_types = ('document', 'paper', 'image') @@ -160,9 +160,9 @@ _cleared = _dispatched - _stamped # scan_corpus = the RAW full corpus so in-root files newly excluded since last run # are dropped rather than masquerading as deletions; untouched rows preserved (#1908). _scan = {f for fl in incremental['files'].values() for f in fl} -save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) print('[graphify update] Manifest saved.') -" +" "INPUT_PATH" ``` Then run Steps 4–8 on the merged graph as normal. diff --git a/graphify/skills/droid/references/transcribe.md b/graphify/skills/droid/references/transcribe.md index b967f83799..bbe8b93e45 100644 --- a/graphify/skills/droid/references/transcribe.md +++ b/graphify/skills/droid/references/transcribe.md @@ -8,13 +8,11 @@ Skip this step entirely if `detect` returned zero `video` files. Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. -**Strategy:** Read the god nodes from `graphify-out/.graphify_detect.json` (or the analysis file if it exists from a previous run). You are already a language model — write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. - -**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` +**Strategy:** If `graphify-out/.graphify_analysis.json` exists from a previous run, read its top god-node labels and write a one-sentence domain hint from them. Otherwise use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` Pass the chosen prompt to Whisper as its initial prompt. No separate API call needed. **Step 1 - Write the Whisper prompt yourself.** -Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: +Read the top god-node labels from previous analysis when available, then compose a short domain hint sentence. With no previous analysis, use the generic fallback above. - Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` - Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` diff --git a/graphify/skills/droid/references/update.md b/graphify/skills/droid/references/update.md index bd8e659d3c..058ca8e6d2 100644 --- a/graphify/skills/droid/references/update.md +++ b/graphify/skills/droid/references/update.md @@ -12,7 +12,7 @@ import sys, json from graphify.detect import detect_incremental, save_manifest from pathlib import Path -result = detect_incremental(Path('INPUT_PATH')) +result = detect_incremental(Path(sys.argv[1])) new_total = result.get('new_total', 0) print(json.dumps(result, indent=2, ensure_ascii=False)) Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") @@ -24,7 +24,7 @@ if deleted: print(f'{len(deleted)} deleted file(s) to prune.') if new_total > 0: print(f'{new_total} new/changed file(s) to re-extract.') -" +" "INPUT_PATH" ``` Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: @@ -84,7 +84,7 @@ Then: ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from pathlib import Path from graphify.build import build_merge from graphify.detect import save_manifest @@ -113,7 +113,7 @@ G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, - root='INPUT_PATH', + root=sys.argv[1], directed=IS_DIRECTED, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -150,7 +150,7 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) # is lost forever (#2015). Mirrors the library extract path # (cli._stamped_manifest_files + clear_semantic + scan_corpus). from graphify.cli import _stamped_manifest_files -_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) +_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path(sys.argv[1])) # Changed semantic files dispatched this run but NOT stamped had their chunk fail # or be omitted; clear any stale semantic_hash so they are re-queued (#1948). _sem_types = ('document', 'paper', 'image') @@ -160,9 +160,9 @@ _cleared = _dispatched - _stamped # scan_corpus = the RAW full corpus so in-root files newly excluded since last run # are dropped rather than masquerading as deletions; untouched rows preserved (#1908). _scan = {f for fl in incremental['files'].values() for f in fl} -save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) print('[graphify update] Manifest saved.') -" +" "INPUT_PATH" ``` Then run Steps 4–8 on the merged graph as normal. diff --git a/graphify/skills/kilo/references/transcribe.md b/graphify/skills/kilo/references/transcribe.md index b967f83799..bbe8b93e45 100644 --- a/graphify/skills/kilo/references/transcribe.md +++ b/graphify/skills/kilo/references/transcribe.md @@ -8,13 +8,11 @@ Skip this step entirely if `detect` returned zero `video` files. Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. -**Strategy:** Read the god nodes from `graphify-out/.graphify_detect.json` (or the analysis file if it exists from a previous run). You are already a language model — write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. - -**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` +**Strategy:** If `graphify-out/.graphify_analysis.json` exists from a previous run, read its top god-node labels and write a one-sentence domain hint from them. Otherwise use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` Pass the chosen prompt to Whisper as its initial prompt. No separate API call needed. **Step 1 - Write the Whisper prompt yourself.** -Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: +Read the top god-node labels from previous analysis when available, then compose a short domain hint sentence. With no previous analysis, use the generic fallback above. - Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` - Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` diff --git a/graphify/skills/kilo/references/update.md b/graphify/skills/kilo/references/update.md index bd8e659d3c..058ca8e6d2 100644 --- a/graphify/skills/kilo/references/update.md +++ b/graphify/skills/kilo/references/update.md @@ -12,7 +12,7 @@ import sys, json from graphify.detect import detect_incremental, save_manifest from pathlib import Path -result = detect_incremental(Path('INPUT_PATH')) +result = detect_incremental(Path(sys.argv[1])) new_total = result.get('new_total', 0) print(json.dumps(result, indent=2, ensure_ascii=False)) Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") @@ -24,7 +24,7 @@ if deleted: print(f'{len(deleted)} deleted file(s) to prune.') if new_total > 0: print(f'{new_total} new/changed file(s) to re-extract.') -" +" "INPUT_PATH" ``` Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: @@ -84,7 +84,7 @@ Then: ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from pathlib import Path from graphify.build import build_merge from graphify.detect import save_manifest @@ -113,7 +113,7 @@ G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, - root='INPUT_PATH', + root=sys.argv[1], directed=IS_DIRECTED, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -150,7 +150,7 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) # is lost forever (#2015). Mirrors the library extract path # (cli._stamped_manifest_files + clear_semantic + scan_corpus). from graphify.cli import _stamped_manifest_files -_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) +_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path(sys.argv[1])) # Changed semantic files dispatched this run but NOT stamped had their chunk fail # or be omitted; clear any stale semantic_hash so they are re-queued (#1948). _sem_types = ('document', 'paper', 'image') @@ -160,9 +160,9 @@ _cleared = _dispatched - _stamped # scan_corpus = the RAW full corpus so in-root files newly excluded since last run # are dropped rather than masquerading as deletions; untouched rows preserved (#1908). _scan = {f for fl in incremental['files'].values() for f in fl} -save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) print('[graphify update] Manifest saved.') -" +" "INPUT_PATH" ``` Then run Steps 4–8 on the merged graph as normal. diff --git a/graphify/skills/kiro/references/transcribe.md b/graphify/skills/kiro/references/transcribe.md index b967f83799..bbe8b93e45 100644 --- a/graphify/skills/kiro/references/transcribe.md +++ b/graphify/skills/kiro/references/transcribe.md @@ -8,13 +8,11 @@ Skip this step entirely if `detect` returned zero `video` files. Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. -**Strategy:** Read the god nodes from `graphify-out/.graphify_detect.json` (or the analysis file if it exists from a previous run). You are already a language model — write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. - -**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` +**Strategy:** If `graphify-out/.graphify_analysis.json` exists from a previous run, read its top god-node labels and write a one-sentence domain hint from them. Otherwise use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` Pass the chosen prompt to Whisper as its initial prompt. No separate API call needed. **Step 1 - Write the Whisper prompt yourself.** -Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: +Read the top god-node labels from previous analysis when available, then compose a short domain hint sentence. With no previous analysis, use the generic fallback above. - Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` - Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` diff --git a/graphify/skills/kiro/references/update.md b/graphify/skills/kiro/references/update.md index bd8e659d3c..058ca8e6d2 100644 --- a/graphify/skills/kiro/references/update.md +++ b/graphify/skills/kiro/references/update.md @@ -12,7 +12,7 @@ import sys, json from graphify.detect import detect_incremental, save_manifest from pathlib import Path -result = detect_incremental(Path('INPUT_PATH')) +result = detect_incremental(Path(sys.argv[1])) new_total = result.get('new_total', 0) print(json.dumps(result, indent=2, ensure_ascii=False)) Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") @@ -24,7 +24,7 @@ if deleted: print(f'{len(deleted)} deleted file(s) to prune.') if new_total > 0: print(f'{new_total} new/changed file(s) to re-extract.') -" +" "INPUT_PATH" ``` Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: @@ -84,7 +84,7 @@ Then: ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from pathlib import Path from graphify.build import build_merge from graphify.detect import save_manifest @@ -113,7 +113,7 @@ G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, - root='INPUT_PATH', + root=sys.argv[1], directed=IS_DIRECTED, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -150,7 +150,7 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) # is lost forever (#2015). Mirrors the library extract path # (cli._stamped_manifest_files + clear_semantic + scan_corpus). from graphify.cli import _stamped_manifest_files -_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) +_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path(sys.argv[1])) # Changed semantic files dispatched this run but NOT stamped had their chunk fail # or be omitted; clear any stale semantic_hash so they are re-queued (#1948). _sem_types = ('document', 'paper', 'image') @@ -160,9 +160,9 @@ _cleared = _dispatched - _stamped # scan_corpus = the RAW full corpus so in-root files newly excluded since last run # are dropped rather than masquerading as deletions; untouched rows preserved (#1908). _scan = {f for fl in incremental['files'].values() for f in fl} -save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) print('[graphify update] Manifest saved.') -" +" "INPUT_PATH" ``` Then run Steps 4–8 on the merged graph as normal. diff --git a/graphify/skills/opencode/references/transcribe.md b/graphify/skills/opencode/references/transcribe.md index b967f83799..bbe8b93e45 100644 --- a/graphify/skills/opencode/references/transcribe.md +++ b/graphify/skills/opencode/references/transcribe.md @@ -8,13 +8,11 @@ Skip this step entirely if `detect` returned zero `video` files. Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. -**Strategy:** Read the god nodes from `graphify-out/.graphify_detect.json` (or the analysis file if it exists from a previous run). You are already a language model — write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. - -**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` +**Strategy:** If `graphify-out/.graphify_analysis.json` exists from a previous run, read its top god-node labels and write a one-sentence domain hint from them. Otherwise use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` Pass the chosen prompt to Whisper as its initial prompt. No separate API call needed. **Step 1 - Write the Whisper prompt yourself.** -Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: +Read the top god-node labels from previous analysis when available, then compose a short domain hint sentence. With no previous analysis, use the generic fallback above. - Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` - Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` diff --git a/graphify/skills/opencode/references/update.md b/graphify/skills/opencode/references/update.md index bd8e659d3c..058ca8e6d2 100644 --- a/graphify/skills/opencode/references/update.md +++ b/graphify/skills/opencode/references/update.md @@ -12,7 +12,7 @@ import sys, json from graphify.detect import detect_incremental, save_manifest from pathlib import Path -result = detect_incremental(Path('INPUT_PATH')) +result = detect_incremental(Path(sys.argv[1])) new_total = result.get('new_total', 0) print(json.dumps(result, indent=2, ensure_ascii=False)) Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") @@ -24,7 +24,7 @@ if deleted: print(f'{len(deleted)} deleted file(s) to prune.') if new_total > 0: print(f'{new_total} new/changed file(s) to re-extract.') -" +" "INPUT_PATH" ``` Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: @@ -84,7 +84,7 @@ Then: ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from pathlib import Path from graphify.build import build_merge from graphify.detect import save_manifest @@ -113,7 +113,7 @@ G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, - root='INPUT_PATH', + root=sys.argv[1], directed=IS_DIRECTED, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -150,7 +150,7 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) # is lost forever (#2015). Mirrors the library extract path # (cli._stamped_manifest_files + clear_semantic + scan_corpus). from graphify.cli import _stamped_manifest_files -_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) +_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path(sys.argv[1])) # Changed semantic files dispatched this run but NOT stamped had their chunk fail # or be omitted; clear any stale semantic_hash so they are re-queued (#1948). _sem_types = ('document', 'paper', 'image') @@ -160,9 +160,9 @@ _cleared = _dispatched - _stamped # scan_corpus = the RAW full corpus so in-root files newly excluded since last run # are dropped rather than masquerading as deletions; untouched rows preserved (#1908). _scan = {f for fl in incremental['files'].values() for f in fl} -save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) print('[graphify update] Manifest saved.') -" +" "INPUT_PATH" ``` Then run Steps 4–8 on the merged graph as normal. diff --git a/graphify/skills/pi/references/transcribe.md b/graphify/skills/pi/references/transcribe.md index b967f83799..bbe8b93e45 100644 --- a/graphify/skills/pi/references/transcribe.md +++ b/graphify/skills/pi/references/transcribe.md @@ -8,13 +8,11 @@ Skip this step entirely if `detect` returned zero `video` files. Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. -**Strategy:** Read the god nodes from `graphify-out/.graphify_detect.json` (or the analysis file if it exists from a previous run). You are already a language model — write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. - -**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` +**Strategy:** If `graphify-out/.graphify_analysis.json` exists from a previous run, read its top god-node labels and write a one-sentence domain hint from them. Otherwise use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` Pass the chosen prompt to Whisper as its initial prompt. No separate API call needed. **Step 1 - Write the Whisper prompt yourself.** -Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: +Read the top god-node labels from previous analysis when available, then compose a short domain hint sentence. With no previous analysis, use the generic fallback above. - Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` - Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` diff --git a/graphify/skills/pi/references/update.md b/graphify/skills/pi/references/update.md index bd8e659d3c..058ca8e6d2 100644 --- a/graphify/skills/pi/references/update.md +++ b/graphify/skills/pi/references/update.md @@ -12,7 +12,7 @@ import sys, json from graphify.detect import detect_incremental, save_manifest from pathlib import Path -result = detect_incremental(Path('INPUT_PATH')) +result = detect_incremental(Path(sys.argv[1])) new_total = result.get('new_total', 0) print(json.dumps(result, indent=2, ensure_ascii=False)) Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") @@ -24,7 +24,7 @@ if deleted: print(f'{len(deleted)} deleted file(s) to prune.') if new_total > 0: print(f'{new_total} new/changed file(s) to re-extract.') -" +" "INPUT_PATH" ``` Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: @@ -84,7 +84,7 @@ Then: ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from pathlib import Path from graphify.build import build_merge from graphify.detect import save_manifest @@ -113,7 +113,7 @@ G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, - root='INPUT_PATH', + root=sys.argv[1], directed=IS_DIRECTED, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -150,7 +150,7 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) # is lost forever (#2015). Mirrors the library extract path # (cli._stamped_manifest_files + clear_semantic + scan_corpus). from graphify.cli import _stamped_manifest_files -_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) +_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path(sys.argv[1])) # Changed semantic files dispatched this run but NOT stamped had their chunk fail # or be omitted; clear any stale semantic_hash so they are re-queued (#1948). _sem_types = ('document', 'paper', 'image') @@ -160,9 +160,9 @@ _cleared = _dispatched - _stamped # scan_corpus = the RAW full corpus so in-root files newly excluded since last run # are dropped rather than masquerading as deletions; untouched rows preserved (#1908). _scan = {f for fl in incremental['files'].values() for f in fl} -save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) print('[graphify update] Manifest saved.') -" +" "INPUT_PATH" ``` Then run Steps 4–8 on the merged graph as normal. diff --git a/graphify/skills/trae/references/transcribe.md b/graphify/skills/trae/references/transcribe.md index b967f83799..bbe8b93e45 100644 --- a/graphify/skills/trae/references/transcribe.md +++ b/graphify/skills/trae/references/transcribe.md @@ -8,13 +8,11 @@ Skip this step entirely if `detect` returned zero `video` files. Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. -**Strategy:** Read the god nodes from `graphify-out/.graphify_detect.json` (or the analysis file if it exists from a previous run). You are already a language model — write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. - -**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` +**Strategy:** If `graphify-out/.graphify_analysis.json` exists from a previous run, read its top god-node labels and write a one-sentence domain hint from them. Otherwise use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` Pass the chosen prompt to Whisper as its initial prompt. No separate API call needed. **Step 1 - Write the Whisper prompt yourself.** -Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: +Read the top god-node labels from previous analysis when available, then compose a short domain hint sentence. With no previous analysis, use the generic fallback above. - Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` - Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` diff --git a/graphify/skills/trae/references/update.md b/graphify/skills/trae/references/update.md index bd8e659d3c..058ca8e6d2 100644 --- a/graphify/skills/trae/references/update.md +++ b/graphify/skills/trae/references/update.md @@ -12,7 +12,7 @@ import sys, json from graphify.detect import detect_incremental, save_manifest from pathlib import Path -result = detect_incremental(Path('INPUT_PATH')) +result = detect_incremental(Path(sys.argv[1])) new_total = result.get('new_total', 0) print(json.dumps(result, indent=2, ensure_ascii=False)) Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") @@ -24,7 +24,7 @@ if deleted: print(f'{len(deleted)} deleted file(s) to prune.') if new_total > 0: print(f'{new_total} new/changed file(s) to re-extract.') -" +" "INPUT_PATH" ``` Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: @@ -84,7 +84,7 @@ Then: ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from pathlib import Path from graphify.build import build_merge from graphify.detect import save_manifest @@ -113,7 +113,7 @@ G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, - root='INPUT_PATH', + root=sys.argv[1], directed=IS_DIRECTED, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -150,7 +150,7 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) # is lost forever (#2015). Mirrors the library extract path # (cli._stamped_manifest_files + clear_semantic + scan_corpus). from graphify.cli import _stamped_manifest_files -_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) +_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path(sys.argv[1])) # Changed semantic files dispatched this run but NOT stamped had their chunk fail # or be omitted; clear any stale semantic_hash so they are re-queued (#1948). _sem_types = ('document', 'paper', 'image') @@ -160,9 +160,9 @@ _cleared = _dispatched - _stamped # scan_corpus = the RAW full corpus so in-root files newly excluded since last run # are dropped rather than masquerading as deletions; untouched rows preserved (#1908). _scan = {f for fl in incremental['files'].values() for f in fl} -save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) print('[graphify update] Manifest saved.') -" +" "INPUT_PATH" ``` Then run Steps 4–8 on the merged graph as normal. diff --git a/graphify/skills/vscode/references/transcribe.md b/graphify/skills/vscode/references/transcribe.md index b967f83799..bbe8b93e45 100644 --- a/graphify/skills/vscode/references/transcribe.md +++ b/graphify/skills/vscode/references/transcribe.md @@ -8,13 +8,11 @@ Skip this step entirely if `detect` returned zero `video` files. Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. -**Strategy:** Read the god nodes from `graphify-out/.graphify_detect.json` (or the analysis file if it exists from a previous run). You are already a language model — write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. - -**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` +**Strategy:** If `graphify-out/.graphify_analysis.json` exists from a previous run, read its top god-node labels and write a one-sentence domain hint from them. Otherwise use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` Pass the chosen prompt to Whisper as its initial prompt. No separate API call needed. **Step 1 - Write the Whisper prompt yourself.** -Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: +Read the top god-node labels from previous analysis when available, then compose a short domain hint sentence. With no previous analysis, use the generic fallback above. - Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` - Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` diff --git a/graphify/skills/vscode/references/update.md b/graphify/skills/vscode/references/update.md index bd8e659d3c..058ca8e6d2 100644 --- a/graphify/skills/vscode/references/update.md +++ b/graphify/skills/vscode/references/update.md @@ -12,7 +12,7 @@ import sys, json from graphify.detect import detect_incremental, save_manifest from pathlib import Path -result = detect_incremental(Path('INPUT_PATH')) +result = detect_incremental(Path(sys.argv[1])) new_total = result.get('new_total', 0) print(json.dumps(result, indent=2, ensure_ascii=False)) Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") @@ -24,7 +24,7 @@ if deleted: print(f'{len(deleted)} deleted file(s) to prune.') if new_total > 0: print(f'{new_total} new/changed file(s) to re-extract.') -" +" "INPUT_PATH" ``` Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: @@ -84,7 +84,7 @@ Then: ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from pathlib import Path from graphify.build import build_merge from graphify.detect import save_manifest @@ -113,7 +113,7 @@ G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, - root='INPUT_PATH', + root=sys.argv[1], directed=IS_DIRECTED, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -150,7 +150,7 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) # is lost forever (#2015). Mirrors the library extract path # (cli._stamped_manifest_files + clear_semantic + scan_corpus). from graphify.cli import _stamped_manifest_files -_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) +_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path(sys.argv[1])) # Changed semantic files dispatched this run but NOT stamped had their chunk fail # or be omitted; clear any stale semantic_hash so they are re-queued (#1948). _sem_types = ('document', 'paper', 'image') @@ -160,9 +160,9 @@ _cleared = _dispatched - _stamped # scan_corpus = the RAW full corpus so in-root files newly excluded since last run # are dropped rather than masquerading as deletions; untouched rows preserved (#1908). _scan = {f for fl in incremental['files'].values() for f in fl} -save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) print('[graphify update] Manifest saved.') -" +" "INPUT_PATH" ``` Then run Steps 4–8 on the merged graph as normal. diff --git a/graphify/skills/windows/references/transcribe.md b/graphify/skills/windows/references/transcribe.md index b967f83799..bbe8b93e45 100644 --- a/graphify/skills/windows/references/transcribe.md +++ b/graphify/skills/windows/references/transcribe.md @@ -8,13 +8,11 @@ Skip this step entirely if `detect` returned zero `video` files. Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. -**Strategy:** Read the god nodes from `graphify-out/.graphify_detect.json` (or the analysis file if it exists from a previous run). You are already a language model — write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. - -**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` +**Strategy:** If `graphify-out/.graphify_analysis.json` exists from a previous run, read its top god-node labels and write a one-sentence domain hint from them. Otherwise use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` Pass the chosen prompt to Whisper as its initial prompt. No separate API call needed. **Step 1 - Write the Whisper prompt yourself.** -Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: +Read the top god-node labels from previous analysis when available, then compose a short domain hint sentence. With no previous analysis, use the generic fallback above. - Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` - Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` diff --git a/graphify/skills/windows/references/update.md b/graphify/skills/windows/references/update.md index bd8e659d3c..058ca8e6d2 100644 --- a/graphify/skills/windows/references/update.md +++ b/graphify/skills/windows/references/update.md @@ -12,7 +12,7 @@ import sys, json from graphify.detect import detect_incremental, save_manifest from pathlib import Path -result = detect_incremental(Path('INPUT_PATH')) +result = detect_incremental(Path(sys.argv[1])) new_total = result.get('new_total', 0) print(json.dumps(result, indent=2, ensure_ascii=False)) Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") @@ -24,7 +24,7 @@ if deleted: print(f'{len(deleted)} deleted file(s) to prune.') if new_total > 0: print(f'{new_total} new/changed file(s) to re-extract.') -" +" "INPUT_PATH" ``` Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: @@ -84,7 +84,7 @@ Then: ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from pathlib import Path from graphify.build import build_merge from graphify.detect import save_manifest @@ -113,7 +113,7 @@ G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, - root='INPUT_PATH', + root=sys.argv[1], directed=IS_DIRECTED, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -150,7 +150,7 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) # is lost forever (#2015). Mirrors the library extract path # (cli._stamped_manifest_files + clear_semantic + scan_corpus). from graphify.cli import _stamped_manifest_files -_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) +_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path(sys.argv[1])) # Changed semantic files dispatched this run but NOT stamped had their chunk fail # or be omitted; clear any stale semantic_hash so they are re-queued (#1948). _sem_types = ('document', 'paper', 'image') @@ -160,9 +160,9 @@ _cleared = _dispatched - _stamped # scan_corpus = the RAW full corpus so in-root files newly excluded since last run # are dropped rather than masquerading as deletions; untouched rows preserved (#1908). _scan = {f for fl in incremental['files'].values() for f in fl} -save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) print('[graphify update] Manifest saved.') -" +" "INPUT_PATH" ``` Then run Steps 4–8 on the merged graph as normal. diff --git a/tests/test_skillgen.py b/tests/test_skillgen.py index 31c454e9ca..0c61152e1e 100644 --- a/tests/test_skillgen.py +++ b/tests/test_skillgen.py @@ -111,6 +111,30 @@ def test_rendered_instructions_preserve_scan_root_and_runnable_commands(): for artifact in watch_artifacts: assert 'graphify.watch "INPUT_PATH"' in artifact.content, artifact.path assert "graphify.watch INPUT_PATH" not in artifact.content, artifact.path + assert "'INPUT_PATH'" not in artifact.content, artifact.path + assert "'SPEC_PATH'" not in artifact.content, artifact.path + + agents_core = next( + artifact + for artifact in artifacts + if artifact.path == "graphify/skill-agents.md" + ) + assert "You MUST use the Agent tool here" not in agents_core.content + assert "After each Agent call completes" not in agents_core.content + assert "You MUST use the subagent tool here" in agents_core.content + assert "After each subagent call completes" in agents_core.content + + transcription_artifacts = [ + artifact + for artifact in artifacts + if artifact.path.endswith("/references/transcribe.md") + or artifact.path in {"graphify/skill-aider.md", "graphify/skill-devin.md"} + ] + assert transcription_artifacts + for artifact in transcription_artifacts: + assert "god nodes from" not in artifact.content, artifact.path + assert "previous analysis" in artifact.content, artifact.path + exports = [ artifact @@ -525,7 +549,7 @@ def test_windows_python_step_bodies_match_posix_verbatim(): for line in claude_core.splitlines(): if line == gen._PY_INVOKE_POSIX: current = [] - elif current is not None and line == '"': + elif current is not None and line in gen._PY_CLOSE_TRANSLATIONS: bodies.append("\n".join(gen._unescape_bash_dq(l) for l in current)) current = None elif current is not None: @@ -553,6 +577,21 @@ def test_powershell_translator_rejects_unknown_bash(): assert gen._translate_bash_block([gen._FIND_CHUNKS_POSIX]) == [gen._FIND_CHUNKS_PS] +def test_powershell_translator_preserves_python_argv(): + """Path placeholders remain shell arguments when Python becomes a here-string.""" + assert gen._translate_bash_block([ + gen._PY_INVOKE_POSIX, + "import sys", + "print(sys.argv[1])", + '" "INPUT_PATH"', + ]) == [ + gen._PY_INVOKE_PS_OPEN, + "import sys", + "print(sys.argv[1])", + gen._PY_INVOKE_PS_CLOSE + ' "INPUT_PATH"', + ] + + def test_posix_hosts_keep_their_bash_invocations(): """The translation is scoped to powershell-shell hosts: the POSIX core keeps its ``$(cat ...) -c`` blocks and bash fences byte-for-byte.""" @@ -1180,12 +1219,18 @@ def test_semantic_cache_calls_pass_prompt_file_for_every_split_host(): for a in bodies: for call in ("check_semantic_cache(", "save_semantic_cache("): line = next(ln for ln in a.content.splitlines() if call in ln and "import" not in ln) - assert "prompt_file='SPEC_PATH'" in line, ( - f"{a.path}: {call} must pass prompt_file so entries are attributed " - f"to the extraction prompt (#1939) — got: {line.strip()}" + assert "prompt_file=sys.argv[2]" in line, ( + f"{a.path}: {call} must read the extraction prompt from argv " + f"(#1939) — got: {line.strip()}" ) # The placeholder is inert unless the body tells the agent what to substitute. assert "SPEC_PATH below is the **absolute** path" in a.content, a.path + argv_close = ( + gen._PY_INVOKE_PS_CLOSE + ' "INPUT_PATH" "SPEC_PATH"' + if a.path == "graphify/skill-windows.md" + else '" "INPUT_PATH" "SPEC_PATH"' + ) + assert a.content.count(argv_close) == 2, a.path def test_windows_skill_writes_marker_files_without_a_bom(): diff --git a/tools/skillgen/expected/graphify__skill-agents.md b/tools/skillgen/expected/graphify__skill-agents.md index 846a316445..78361e1bde 100644 --- a/tools/skillgen/expected/graphify__skill-agents.md +++ b/tools/skillgen/expected/graphify__skill-agents.md @@ -108,15 +108,15 @@ If the import succeeds, print nothing and move straight to Step 2. ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from graphify.detect import detect from pathlib import Path -result = detect(Path('INPUT_PATH')) +result = detect(Path(sys.argv[1])) # Write the sidecar from Python, not a shell redirect, so the same block renders # on PowerShell hosts without console-encoding drift (#2528). Path('graphify-out/.graphify_detect.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") print(f'Detected {result[\"total_files\"]} files') -" +" "INPUT_PATH" ``` Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: @@ -183,13 +183,13 @@ for f in detect.get('files', {}).get('code', []): code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) if code_files: - result = extract(code_files, cache_root=Path('INPUT_PATH')) + result = extract(code_files, cache_root=Path(sys.argv[1])) Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') else: Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") print('No code files - skipping AST extraction') -" +" "INPUT_PATH" ``` #### Part B - Semantic extraction (parallel subagents) @@ -204,7 +204,7 @@ Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],' " ``` -**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** +**MANDATORY: You MUST use the subagent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the subagent tool you are doing this wrong.** Before dispatching subagents, print a timing estimate: - Load `total_words` and file counts from `graphify-out/.graphify_detect.json` @@ -220,7 +220,7 @@ SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from graphify.cache import check_semantic_cache from pathlib import Path @@ -230,7 +230,7 @@ detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encodin # every source file (#1392). Video is transcribed to a document in Step 2.5 first. all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root=sys.argv[1], prompt_file=sys.argv[2]) # Always (re)write the cache file: write hits, else DELETE any leftover from a prior # run so Part C never merges a stale .graphify_cached.json (#1392). @@ -240,7 +240,7 @@ else: Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" +" "INPUT_PATH" "SPEC_PATH" ``` Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. @@ -282,7 +282,7 @@ Wait for all subagents. For each result: If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: +Merge all chunk files into `.graphify_semantic_new.json`. **After each subagent call completes, read the real token counts from the subagent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: ```bash $(cat graphify-out/.graphify_python) -c " import json, glob @@ -309,15 +309,15 @@ print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens' Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root=sys.argv[1], allowed_source_files=uncached, prompt_file=sys.argv[2]) print(f'Cached {saved} files') -" +" "INPUT_PATH" "SPEC_PATH" ``` Merge cached + new results into `graphify-out/.graphify_semantic.json`: @@ -406,7 +406,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -431,7 +431,7 @@ if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') print('If this shrink is intentional (you deleted files), re-run a full build with --force.') raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, sys.argv[1], suggested_questions=questions) Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") analysis = { 'communities': {str(k): v for k, v in communities.items()}, @@ -442,7 +442,7 @@ analysis = { } Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" +" "INPUT_PATH" ``` If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. @@ -455,12 +455,12 @@ A non-destructive diagnostic on the extraction, before labeling. It surfaces edg ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from pathlib import Path from graphify.diagnostics import diagnose_extraction, format_diagnostic_report extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') +summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root=sys.argv[1]) print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( ('dangling_endpoint_edges', 'dangling-endpoint edges'), @@ -470,7 +470,7 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0)] print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') -" +" "INPUT_PATH" ``` Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules). @@ -496,7 +496,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} @@ -507,7 +507,7 @@ labels = LABELS_DICT # Regenerate questions with real community labels (labels affect question phrasing) questions = suggest_questions(G, communities, labels) -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, sys.argv[1], suggested_questions=questions) # Re-export so graph.json nodes carry the curated community_name (#2490). # Same extraction as Step 4, so the #479 shrink-guard passes on node count; # if it still refuses, surface the guard message - do not force past it. @@ -519,7 +519,7 @@ if not wrote: Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") print('Report updated with community labels') -" +" "INPUT_PATH" ``` Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). @@ -554,7 +554,7 @@ These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, ` ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from pathlib import Path from datetime import datetime, timezone from graphify.detect import save_manifest @@ -577,7 +577,7 @@ extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encod # types are gated on output. from graphify.cli import _stamped_manifest_files _corpus = detect.get('all_files') or detect['files'] -_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) +_manifest_files = _stamped_manifest_files(_corpus, extract, Path(sys.argv[1])) # Files dispatched this run (the changed subset) but NOT stamped above still carry # a stale semantic_hash from a prior run; clear it so detect_incremental re-queues # them instead of reading them as unchanged (#1948). @@ -589,7 +589,7 @@ _cleared = _dispatched - _stamped # files newly excluded since last run are dropped rather than masquerading as # deletions; untouched files' prior rows are still preserved (#1908). _scan = {f for fl in _corpus.values() for f in fl} -save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) # Update cumulative cost tracker input_tok = extract.get('input_tokens', 0) @@ -613,7 +613,7 @@ cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\" print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" +" "INPUT_PATH" rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null rm -f graphify-out/needs_update 2>/dev/null || true diff --git a/tools/skillgen/expected/graphify__skill-aider.md b/tools/skillgen/expected/graphify__skill-aider.md index 4c5f74c88b..085be97cd2 100644 --- a/tools/skillgen/expected/graphify__skill-aider.md +++ b/tools/skillgen/expected/graphify__skill-aider.md @@ -101,11 +101,12 @@ If the import succeeds, print nothing and move straight to Step 2. ```bash $(cat graphify-out/.graphify_python) -c " import json +import sys from graphify.detect import detect from pathlib import Path -result = detect(Path('INPUT_PATH')) +result = detect(Path(sys.argv[1])) print(json.dumps(result)) -" > .graphify_detect.json +" "INPUT_PATH" > .graphify_detect.json ``` Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: @@ -133,13 +134,11 @@ Skip this step entirely if `detect` returned zero `video` files. Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. -**Strategy:** Read the god nodes from the detect output or analysis file. You are already a language model - write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. - -**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` +**Strategy:** If an analysis file exists from a previous run, read its top god-node labels and write a one-sentence domain hint from them. Otherwise use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` Pass the chosen prompt to Whisper as its initial prompt. No separate API call needed. **Step 1 - Write the Whisper prompt yourself.** -Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: +Read the top god-node labels from previous analysis when available, then compose a short domain hint sentence. With no previous analysis, use the generic fallback above. - Labels: `transformer, attention, encoder, decoder` -> `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` - Labels: `kubernetes, deployment, pod, helm` -> `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` @@ -426,7 +425,7 @@ wrote = to_json(G, communities, 'graphify-out/graph.json') if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (fewer nodes than the existing graph). Run a full rebuild to be safe.') raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, sys.argv[1], suggested_questions=questions) Path('graphify-out/GRAPH_REPORT.md').write_text(report) analysis = { @@ -438,7 +437,7 @@ analysis = { } Path('.graphify_analysis.json').write_text(json.dumps(analysis, indent=2)) print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" +" "INPUT_PATH" ``` If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. @@ -476,7 +475,7 @@ labels = LABELS_DICT # Regenerate questions with real community labels (labels affect question phrasing) questions = suggest_questions(G, communities, labels) -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, sys.argv[1], suggested_questions=questions) # Re-export so graph.json nodes carry the curated community_name (#2490). # Same extraction as Step 4, so the #479 shrink-guard passes on node count; # if it still refuses, surface the guard message - do not force past it. @@ -487,7 +486,7 @@ if not wrote: Path('graphify-out/GRAPH_REPORT.md').write_text(report) Path('.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()})) print('Report updated with community labels') -" +" "INPUT_PATH" ``` Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). @@ -679,6 +678,7 @@ Print the output directly in chat. If `total_words <= 5000`, skip silently - the ```bash $(cat graphify-out/.graphify_python) -c " import json +import sys from pathlib import Path from datetime import datetime, timezone from graphify.detect import save_manifest @@ -689,13 +689,13 @@ extract = json.loads(Path('.graphify_extract.json').read_text()) # Stamp only semantic files that produced output so a failed chunk is re-queued next run, not lost (#2015). from graphify.cli import _stamped_manifest_files _corpus = detect.get('all_files') or detect['files'] -_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) +_manifest_files = _stamped_manifest_files(_corpus, extract, Path(sys.argv[1])) _sem_types = ('document', 'paper', 'image') _dispatched = {f for t, fl in detect['files'].items() if t in _sem_types for f in fl} _stamped = {f for fl in _manifest_files.values() for f in fl} _cleared = _dispatched - _stamped _scan = {f for fl in _corpus.values() for f in fl} -save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) # Update cumulative cost tracker input_tok = extract.get('input_tokens', 0) @@ -719,7 +719,7 @@ cost_path.write_text(json.dumps(cost, indent=2)) print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" +" "INPUT_PATH" rm -f .graphify_detect.json .graphify_extract.json .graphify_ast.json .graphify_semantic.json .graphify_analysis.json .graphify_labels.json; find . -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null rm -f graphify-out/needs_update 2>/dev/null || true ``` @@ -765,7 +765,7 @@ import sys, json from graphify.detect import detect_incremental, save_manifest from pathlib import Path -result = detect_incremental(Path('INPUT_PATH')) +result = detect_incremental(Path(sys.argv[1])) new_total = result.get('new_total', 0) print(json.dumps(result, indent=2)) Path('.graphify_incremental.json').write_text(json.dumps(result)) @@ -777,7 +777,7 @@ if deleted: print(f'{len(deleted)} deleted file(s) to prune.') if new_total > 0: print(f'{new_total} new/changed file(s) to re-extract.') -" +" "INPUT_PATH" ``` If new files exist, first check whether all changed files are code files: diff --git a/tools/skillgen/expected/graphify__skill-amp.md b/tools/skillgen/expected/graphify__skill-amp.md index 846a316445..78361e1bde 100644 --- a/tools/skillgen/expected/graphify__skill-amp.md +++ b/tools/skillgen/expected/graphify__skill-amp.md @@ -108,15 +108,15 @@ If the import succeeds, print nothing and move straight to Step 2. ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from graphify.detect import detect from pathlib import Path -result = detect(Path('INPUT_PATH')) +result = detect(Path(sys.argv[1])) # Write the sidecar from Python, not a shell redirect, so the same block renders # on PowerShell hosts without console-encoding drift (#2528). Path('graphify-out/.graphify_detect.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") print(f'Detected {result[\"total_files\"]} files') -" +" "INPUT_PATH" ``` Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: @@ -183,13 +183,13 @@ for f in detect.get('files', {}).get('code', []): code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) if code_files: - result = extract(code_files, cache_root=Path('INPUT_PATH')) + result = extract(code_files, cache_root=Path(sys.argv[1])) Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') else: Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") print('No code files - skipping AST extraction') -" +" "INPUT_PATH" ``` #### Part B - Semantic extraction (parallel subagents) @@ -204,7 +204,7 @@ Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],' " ``` -**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** +**MANDATORY: You MUST use the subagent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the subagent tool you are doing this wrong.** Before dispatching subagents, print a timing estimate: - Load `total_words` and file counts from `graphify-out/.graphify_detect.json` @@ -220,7 +220,7 @@ SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from graphify.cache import check_semantic_cache from pathlib import Path @@ -230,7 +230,7 @@ detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encodin # every source file (#1392). Video is transcribed to a document in Step 2.5 first. all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root=sys.argv[1], prompt_file=sys.argv[2]) # Always (re)write the cache file: write hits, else DELETE any leftover from a prior # run so Part C never merges a stale .graphify_cached.json (#1392). @@ -240,7 +240,7 @@ else: Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" +" "INPUT_PATH" "SPEC_PATH" ``` Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. @@ -282,7 +282,7 @@ Wait for all subagents. For each result: If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: +Merge all chunk files into `.graphify_semantic_new.json`. **After each subagent call completes, read the real token counts from the subagent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: ```bash $(cat graphify-out/.graphify_python) -c " import json, glob @@ -309,15 +309,15 @@ print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens' Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root=sys.argv[1], allowed_source_files=uncached, prompt_file=sys.argv[2]) print(f'Cached {saved} files') -" +" "INPUT_PATH" "SPEC_PATH" ``` Merge cached + new results into `graphify-out/.graphify_semantic.json`: @@ -406,7 +406,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -431,7 +431,7 @@ if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') print('If this shrink is intentional (you deleted files), re-run a full build with --force.') raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, sys.argv[1], suggested_questions=questions) Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") analysis = { 'communities': {str(k): v for k, v in communities.items()}, @@ -442,7 +442,7 @@ analysis = { } Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" +" "INPUT_PATH" ``` If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. @@ -455,12 +455,12 @@ A non-destructive diagnostic on the extraction, before labeling. It surfaces edg ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from pathlib import Path from graphify.diagnostics import diagnose_extraction, format_diagnostic_report extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') +summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root=sys.argv[1]) print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( ('dangling_endpoint_edges', 'dangling-endpoint edges'), @@ -470,7 +470,7 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0)] print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') -" +" "INPUT_PATH" ``` Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules). @@ -496,7 +496,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} @@ -507,7 +507,7 @@ labels = LABELS_DICT # Regenerate questions with real community labels (labels affect question phrasing) questions = suggest_questions(G, communities, labels) -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, sys.argv[1], suggested_questions=questions) # Re-export so graph.json nodes carry the curated community_name (#2490). # Same extraction as Step 4, so the #479 shrink-guard passes on node count; # if it still refuses, surface the guard message - do not force past it. @@ -519,7 +519,7 @@ if not wrote: Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") print('Report updated with community labels') -" +" "INPUT_PATH" ``` Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). @@ -554,7 +554,7 @@ These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, ` ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from pathlib import Path from datetime import datetime, timezone from graphify.detect import save_manifest @@ -577,7 +577,7 @@ extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encod # types are gated on output. from graphify.cli import _stamped_manifest_files _corpus = detect.get('all_files') or detect['files'] -_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) +_manifest_files = _stamped_manifest_files(_corpus, extract, Path(sys.argv[1])) # Files dispatched this run (the changed subset) but NOT stamped above still carry # a stale semantic_hash from a prior run; clear it so detect_incremental re-queues # them instead of reading them as unchanged (#1948). @@ -589,7 +589,7 @@ _cleared = _dispatched - _stamped # files newly excluded since last run are dropped rather than masquerading as # deletions; untouched files' prior rows are still preserved (#1908). _scan = {f for fl in _corpus.values() for f in fl} -save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) # Update cumulative cost tracker input_tok = extract.get('input_tokens', 0) @@ -613,7 +613,7 @@ cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\" print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" +" "INPUT_PATH" rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null rm -f graphify-out/needs_update 2>/dev/null || true diff --git a/tools/skillgen/expected/graphify__skill-claw.md b/tools/skillgen/expected/graphify__skill-claw.md index 56c5c75fc2..7e7c797563 100644 --- a/tools/skillgen/expected/graphify__skill-claw.md +++ b/tools/skillgen/expected/graphify__skill-claw.md @@ -108,15 +108,15 @@ If the import succeeds, print nothing and move straight to Step 2. ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from graphify.detect import detect from pathlib import Path -result = detect(Path('INPUT_PATH')) +result = detect(Path(sys.argv[1])) # Write the sidecar from Python, not a shell redirect, so the same block renders # on PowerShell hosts without console-encoding drift (#2528). Path('graphify-out/.graphify_detect.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") print(f'Detected {result[\"total_files\"]} files') -" +" "INPUT_PATH" ``` Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: @@ -183,13 +183,13 @@ for f in detect.get('files', {}).get('code', []): code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) if code_files: - result = extract(code_files, cache_root=Path('INPUT_PATH')) + result = extract(code_files, cache_root=Path(sys.argv[1])) Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') else: Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") print('No code files - skipping AST extraction') -" +" "INPUT_PATH" ``` #### Part B - Semantic extraction (parallel subagents) @@ -204,7 +204,7 @@ Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],' " ``` -**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** +**MANDATORY: You MUST use the subagent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the subagent tool you are doing this wrong.** Before dispatching subagents, print a timing estimate: - Load `total_words` and file counts from `graphify-out/.graphify_detect.json` @@ -220,7 +220,7 @@ SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from graphify.cache import check_semantic_cache from pathlib import Path @@ -230,7 +230,7 @@ detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encodin # every source file (#1392). Video is transcribed to a document in Step 2.5 first. all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root=sys.argv[1], prompt_file=sys.argv[2]) # Always (re)write the cache file: write hits, else DELETE any leftover from a prior # run so Part C never merges a stale .graphify_cached.json (#1392). @@ -240,7 +240,7 @@ else: Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" +" "INPUT_PATH" "SPEC_PATH" ``` Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. @@ -285,7 +285,7 @@ Wait for all subagents. For each result: If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: +Merge all chunk files into `.graphify_semantic_new.json`. **After each subagent call completes, read the real token counts from the subagent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: ```bash $(cat graphify-out/.graphify_python) -c " import json, glob @@ -312,15 +312,15 @@ print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens' Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root=sys.argv[1], allowed_source_files=uncached, prompt_file=sys.argv[2]) print(f'Cached {saved} files') -" +" "INPUT_PATH" "SPEC_PATH" ``` Merge cached + new results into `graphify-out/.graphify_semantic.json`: @@ -409,7 +409,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -434,7 +434,7 @@ if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') print('If this shrink is intentional (you deleted files), re-run a full build with --force.') raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, sys.argv[1], suggested_questions=questions) Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") analysis = { 'communities': {str(k): v for k, v in communities.items()}, @@ -445,7 +445,7 @@ analysis = { } Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" +" "INPUT_PATH" ``` If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. @@ -458,12 +458,12 @@ A non-destructive diagnostic on the extraction, before labeling. It surfaces edg ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from pathlib import Path from graphify.diagnostics import diagnose_extraction, format_diagnostic_report extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') +summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root=sys.argv[1]) print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( ('dangling_endpoint_edges', 'dangling-endpoint edges'), @@ -473,7 +473,7 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0)] print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') -" +" "INPUT_PATH" ``` Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules). @@ -499,7 +499,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} @@ -510,7 +510,7 @@ labels = LABELS_DICT # Regenerate questions with real community labels (labels affect question phrasing) questions = suggest_questions(G, communities, labels) -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, sys.argv[1], suggested_questions=questions) # Re-export so graph.json nodes carry the curated community_name (#2490). # Same extraction as Step 4, so the #479 shrink-guard passes on node count; # if it still refuses, surface the guard message - do not force past it. @@ -522,7 +522,7 @@ if not wrote: Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") print('Report updated with community labels') -" +" "INPUT_PATH" ``` Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). @@ -557,7 +557,7 @@ These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, ` ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from pathlib import Path from datetime import datetime, timezone from graphify.detect import save_manifest @@ -580,7 +580,7 @@ extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encod # types are gated on output. from graphify.cli import _stamped_manifest_files _corpus = detect.get('all_files') or detect['files'] -_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) +_manifest_files = _stamped_manifest_files(_corpus, extract, Path(sys.argv[1])) # Files dispatched this run (the changed subset) but NOT stamped above still carry # a stale semantic_hash from a prior run; clear it so detect_incremental re-queues # them instead of reading them as unchanged (#1948). @@ -592,7 +592,7 @@ _cleared = _dispatched - _stamped # files newly excluded since last run are dropped rather than masquerading as # deletions; untouched files' prior rows are still preserved (#1908). _scan = {f for fl in _corpus.values() for f in fl} -save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) # Update cumulative cost tracker input_tok = extract.get('input_tokens', 0) @@ -616,7 +616,7 @@ cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\" print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" +" "INPUT_PATH" rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null rm -f graphify-out/needs_update 2>/dev/null || true diff --git a/tools/skillgen/expected/graphify__skill-codex.md b/tools/skillgen/expected/graphify__skill-codex.md index 4b23633542..5e085f0a6f 100644 --- a/tools/skillgen/expected/graphify__skill-codex.md +++ b/tools/skillgen/expected/graphify__skill-codex.md @@ -108,15 +108,15 @@ If the import succeeds, print nothing and move straight to Step 2. ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from graphify.detect import detect from pathlib import Path -result = detect(Path('INPUT_PATH')) +result = detect(Path(sys.argv[1])) # Write the sidecar from Python, not a shell redirect, so the same block renders # on PowerShell hosts without console-encoding drift (#2528). Path('graphify-out/.graphify_detect.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") print(f'Detected {result[\"total_files\"]} files') -" +" "INPUT_PATH" ``` Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: @@ -183,13 +183,13 @@ for f in detect.get('files', {}).get('code', []): code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) if code_files: - result = extract(code_files, cache_root=Path('INPUT_PATH')) + result = extract(code_files, cache_root=Path(sys.argv[1])) Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') else: Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") print('No code files - skipping AST extraction') -" +" "INPUT_PATH" ``` #### Part B - Semantic extraction (parallel subagents) @@ -204,7 +204,7 @@ Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],' " ``` -**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** +**MANDATORY: You MUST use the subagent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the subagent tool you are doing this wrong.** Before dispatching subagents, print a timing estimate: - Load `total_words` and file counts from `graphify-out/.graphify_detect.json` @@ -220,7 +220,7 @@ SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from graphify.cache import check_semantic_cache from pathlib import Path @@ -230,7 +230,7 @@ detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encodin # every source file (#1392). Video is transcribed to a document in Step 2.5 first. all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root=sys.argv[1], prompt_file=sys.argv[2]) # Always (re)write the cache file: write hits, else DELETE any leftover from a prior # run so Part C never merges a stale .graphify_cached.json (#1392). @@ -240,7 +240,7 @@ else: Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" +" "INPUT_PATH" "SPEC_PATH" ``` Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. @@ -282,7 +282,7 @@ Wait for all subagents. For each result: If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: +Merge all chunk files into `.graphify_semantic_new.json`. **After each subagent call completes, read the real token counts from the subagent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: ```bash $(cat graphify-out/.graphify_python) -c " import json, glob @@ -309,15 +309,15 @@ print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens' Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root=sys.argv[1], allowed_source_files=uncached, prompt_file=sys.argv[2]) print(f'Cached {saved} files') -" +" "INPUT_PATH" "SPEC_PATH" ``` Merge cached + new results into `graphify-out/.graphify_semantic.json`: @@ -406,7 +406,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -431,7 +431,7 @@ if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') print('If this shrink is intentional (you deleted files), re-run a full build with --force.') raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, sys.argv[1], suggested_questions=questions) Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") analysis = { 'communities': {str(k): v for k, v in communities.items()}, @@ -442,7 +442,7 @@ analysis = { } Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" +" "INPUT_PATH" ``` If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. @@ -455,12 +455,12 @@ A non-destructive diagnostic on the extraction, before labeling. It surfaces edg ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from pathlib import Path from graphify.diagnostics import diagnose_extraction, format_diagnostic_report extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') +summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root=sys.argv[1]) print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( ('dangling_endpoint_edges', 'dangling-endpoint edges'), @@ -470,7 +470,7 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0)] print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') -" +" "INPUT_PATH" ``` Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules). @@ -496,7 +496,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} @@ -507,7 +507,7 @@ labels = LABELS_DICT # Regenerate questions with real community labels (labels affect question phrasing) questions = suggest_questions(G, communities, labels) -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, sys.argv[1], suggested_questions=questions) # Re-export so graph.json nodes carry the curated community_name (#2490). # Same extraction as Step 4, so the #479 shrink-guard passes on node count; # if it still refuses, surface the guard message - do not force past it. @@ -519,7 +519,7 @@ if not wrote: Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") print('Report updated with community labels') -" +" "INPUT_PATH" ``` Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). @@ -554,7 +554,7 @@ These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, ` ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from pathlib import Path from datetime import datetime, timezone from graphify.detect import save_manifest @@ -577,7 +577,7 @@ extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encod # types are gated on output. from graphify.cli import _stamped_manifest_files _corpus = detect.get('all_files') or detect['files'] -_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) +_manifest_files = _stamped_manifest_files(_corpus, extract, Path(sys.argv[1])) # Files dispatched this run (the changed subset) but NOT stamped above still carry # a stale semantic_hash from a prior run; clear it so detect_incremental re-queues # them instead of reading them as unchanged (#1948). @@ -589,7 +589,7 @@ _cleared = _dispatched - _stamped # files newly excluded since last run are dropped rather than masquerading as # deletions; untouched files' prior rows are still preserved (#1908). _scan = {f for fl in _corpus.values() for f in fl} -save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) # Update cumulative cost tracker input_tok = extract.get('input_tokens', 0) @@ -613,7 +613,7 @@ cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\" print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" +" "INPUT_PATH" rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null rm -f graphify-out/needs_update 2>/dev/null || true diff --git a/tools/skillgen/expected/graphify__skill-copilot.md b/tools/skillgen/expected/graphify__skill-copilot.md index 56c5c75fc2..7e7c797563 100644 --- a/tools/skillgen/expected/graphify__skill-copilot.md +++ b/tools/skillgen/expected/graphify__skill-copilot.md @@ -108,15 +108,15 @@ If the import succeeds, print nothing and move straight to Step 2. ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from graphify.detect import detect from pathlib import Path -result = detect(Path('INPUT_PATH')) +result = detect(Path(sys.argv[1])) # Write the sidecar from Python, not a shell redirect, so the same block renders # on PowerShell hosts without console-encoding drift (#2528). Path('graphify-out/.graphify_detect.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") print(f'Detected {result[\"total_files\"]} files') -" +" "INPUT_PATH" ``` Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: @@ -183,13 +183,13 @@ for f in detect.get('files', {}).get('code', []): code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) if code_files: - result = extract(code_files, cache_root=Path('INPUT_PATH')) + result = extract(code_files, cache_root=Path(sys.argv[1])) Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') else: Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") print('No code files - skipping AST extraction') -" +" "INPUT_PATH" ``` #### Part B - Semantic extraction (parallel subagents) @@ -204,7 +204,7 @@ Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],' " ``` -**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** +**MANDATORY: You MUST use the subagent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the subagent tool you are doing this wrong.** Before dispatching subagents, print a timing estimate: - Load `total_words` and file counts from `graphify-out/.graphify_detect.json` @@ -220,7 +220,7 @@ SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from graphify.cache import check_semantic_cache from pathlib import Path @@ -230,7 +230,7 @@ detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encodin # every source file (#1392). Video is transcribed to a document in Step 2.5 first. all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root=sys.argv[1], prompt_file=sys.argv[2]) # Always (re)write the cache file: write hits, else DELETE any leftover from a prior # run so Part C never merges a stale .graphify_cached.json (#1392). @@ -240,7 +240,7 @@ else: Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" +" "INPUT_PATH" "SPEC_PATH" ``` Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. @@ -285,7 +285,7 @@ Wait for all subagents. For each result: If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: +Merge all chunk files into `.graphify_semantic_new.json`. **After each subagent call completes, read the real token counts from the subagent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: ```bash $(cat graphify-out/.graphify_python) -c " import json, glob @@ -312,15 +312,15 @@ print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens' Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root=sys.argv[1], allowed_source_files=uncached, prompt_file=sys.argv[2]) print(f'Cached {saved} files') -" +" "INPUT_PATH" "SPEC_PATH" ``` Merge cached + new results into `graphify-out/.graphify_semantic.json`: @@ -409,7 +409,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -434,7 +434,7 @@ if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') print('If this shrink is intentional (you deleted files), re-run a full build with --force.') raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, sys.argv[1], suggested_questions=questions) Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") analysis = { 'communities': {str(k): v for k, v in communities.items()}, @@ -445,7 +445,7 @@ analysis = { } Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" +" "INPUT_PATH" ``` If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. @@ -458,12 +458,12 @@ A non-destructive diagnostic on the extraction, before labeling. It surfaces edg ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from pathlib import Path from graphify.diagnostics import diagnose_extraction, format_diagnostic_report extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') +summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root=sys.argv[1]) print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( ('dangling_endpoint_edges', 'dangling-endpoint edges'), @@ -473,7 +473,7 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0)] print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') -" +" "INPUT_PATH" ``` Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules). @@ -499,7 +499,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} @@ -510,7 +510,7 @@ labels = LABELS_DICT # Regenerate questions with real community labels (labels affect question phrasing) questions = suggest_questions(G, communities, labels) -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, sys.argv[1], suggested_questions=questions) # Re-export so graph.json nodes carry the curated community_name (#2490). # Same extraction as Step 4, so the #479 shrink-guard passes on node count; # if it still refuses, surface the guard message - do not force past it. @@ -522,7 +522,7 @@ if not wrote: Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") print('Report updated with community labels') -" +" "INPUT_PATH" ``` Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). @@ -557,7 +557,7 @@ These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, ` ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from pathlib import Path from datetime import datetime, timezone from graphify.detect import save_manifest @@ -580,7 +580,7 @@ extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encod # types are gated on output. from graphify.cli import _stamped_manifest_files _corpus = detect.get('all_files') or detect['files'] -_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) +_manifest_files = _stamped_manifest_files(_corpus, extract, Path(sys.argv[1])) # Files dispatched this run (the changed subset) but NOT stamped above still carry # a stale semantic_hash from a prior run; clear it so detect_incremental re-queues # them instead of reading them as unchanged (#1948). @@ -592,7 +592,7 @@ _cleared = _dispatched - _stamped # files newly excluded since last run are dropped rather than masquerading as # deletions; untouched files' prior rows are still preserved (#1908). _scan = {f for fl in _corpus.values() for f in fl} -save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) # Update cumulative cost tracker input_tok = extract.get('input_tokens', 0) @@ -616,7 +616,7 @@ cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\" print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" +" "INPUT_PATH" rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null rm -f graphify-out/needs_update 2>/dev/null || true diff --git a/tools/skillgen/expected/graphify__skill-devin.md b/tools/skillgen/expected/graphify__skill-devin.md index cfb5aa7b13..bea72f7eea 100644 --- a/tools/skillgen/expected/graphify__skill-devin.md +++ b/tools/skillgen/expected/graphify__skill-devin.md @@ -114,11 +114,12 @@ If the import succeeds, print nothing and move straight to Step 2. ```bash $(cat graphify-out/.graphify_python) -c " import json +import sys from graphify.detect import detect from pathlib import Path -result = detect(Path('INPUT_PATH')) +result = detect(Path(sys.argv[1])) print(json.dumps(result)) -" > graphify-out/.graphify_detect.json +" "INPUT_PATH" > graphify-out/.graphify_detect.json ``` Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: @@ -146,13 +147,11 @@ Skip this step entirely if `detect` returned zero `video` files. Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. -**Strategy:** Read the god nodes from the detect output or analysis file. You are already a language model - write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. - -**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` +**Strategy:** If an analysis file exists from a previous run, read its top god-node labels and write a one-sentence domain hint from them. Otherwise use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` Pass the chosen prompt to Whisper as its initial prompt. No separate API call needed. **Step 1 - Write the Whisper prompt yourself.** -Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: +Read the top god-node labels from previous analysis when available, then compose a short domain hint sentence. With no previous analysis, use the generic fallback above. - Labels: `transformer, attention, encoder, decoder` -> `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` - Labels: `kubernetes, deployment, pod, helm` -> `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` @@ -491,7 +490,7 @@ wrote = to_json(G, communities, 'graphify-out/graph.json') if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (fewer nodes than the existing graph). Run a full rebuild to be safe.') raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, sys.argv[1], suggested_questions=questions) Path('graphify-out/GRAPH_REPORT.md').write_text(report) analysis = { @@ -503,7 +502,7 @@ analysis = { } Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2)) print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" +" "INPUT_PATH" ``` If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. @@ -541,7 +540,7 @@ labels = LABELS_DICT # Regenerate questions with real community labels (labels affect question phrasing) questions = suggest_questions(G, communities, labels) -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, sys.argv[1], suggested_questions=questions) # Re-export so graph.json nodes carry the curated community_name (#2490). # Same extraction as Step 4, so the #479 shrink-guard passes on node count; # if it still refuses, surface the guard message - do not force past it. @@ -552,7 +551,7 @@ if not wrote: Path('graphify-out/GRAPH_REPORT.md').write_text(report) Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()})) print('Report updated with community labels') -" +" "INPUT_PATH" ``` Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). @@ -797,6 +796,7 @@ Print the output directly in chat. If `total_words <= 5000`, skip silently - the ```bash $(cat graphify-out/.graphify_python) -c " import json +import sys from pathlib import Path from datetime import datetime, timezone from graphify.detect import save_manifest @@ -807,13 +807,13 @@ extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) # Stamp only semantic files that produced output so a failed chunk is re-queued next run, not lost (#2015). from graphify.cli import _stamped_manifest_files _corpus = detect.get('all_files') or detect['files'] -_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) +_manifest_files = _stamped_manifest_files(_corpus, extract, Path(sys.argv[1])) _sem_types = ('document', 'paper', 'image') _dispatched = {f for t, fl in detect['files'].items() if t in _sem_types for f in fl} _stamped = {f for fl in _manifest_files.values() for f in fl} _cleared = _dispatched - _stamped _scan = {f for fl in _corpus.values() for f in fl} -save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) # Update cumulative cost tracker input_tok = extract.get('input_tokens', 0) @@ -837,7 +837,7 @@ cost_path.write_text(json.dumps(cost, indent=2)) print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" +" "INPUT_PATH" rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json graphify-out/.graphify_labels.json graphify-out/.graphify_incremental.json graphify-out/.graphify_transcripts.json graphify-out/.graphify_old.json; find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null rm -f graphify-out/needs_update 2>/dev/null || true ``` @@ -904,7 +904,7 @@ import sys, json from graphify.detect import detect_incremental, save_manifest from pathlib import Path -result = detect_incremental(Path('INPUT_PATH')) +result = detect_incremental(Path(sys.argv[1])) new_total = result.get('new_total', 0) print(json.dumps(result, indent=2)) Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result)) @@ -916,7 +916,7 @@ if deleted: print(f'{len(deleted)} deleted file(s) to prune.') if new_total > 0: print(f'{new_total} new/changed file(s) to re-extract.') -" +" "INPUT_PATH" ``` If new files exist, first check whether all changed files are code files: diff --git a/tools/skillgen/expected/graphify__skill-droid.md b/tools/skillgen/expected/graphify__skill-droid.md index f32aac733d..2a896a6fa1 100644 --- a/tools/skillgen/expected/graphify__skill-droid.md +++ b/tools/skillgen/expected/graphify__skill-droid.md @@ -108,15 +108,15 @@ If the import succeeds, print nothing and move straight to Step 2. ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from graphify.detect import detect from pathlib import Path -result = detect(Path('INPUT_PATH')) +result = detect(Path(sys.argv[1])) # Write the sidecar from Python, not a shell redirect, so the same block renders # on PowerShell hosts without console-encoding drift (#2528). Path('graphify-out/.graphify_detect.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") print(f'Detected {result[\"total_files\"]} files') -" +" "INPUT_PATH" ``` Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: @@ -183,13 +183,13 @@ for f in detect.get('files', {}).get('code', []): code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) if code_files: - result = extract(code_files, cache_root=Path('INPUT_PATH')) + result = extract(code_files, cache_root=Path(sys.argv[1])) Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') else: Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") print('No code files - skipping AST extraction') -" +" "INPUT_PATH" ``` #### Part B - Semantic extraction (parallel subagents) @@ -204,7 +204,7 @@ Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],' " ``` -**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** +**MANDATORY: You MUST use the subagent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the subagent tool you are doing this wrong.** Before dispatching subagents, print a timing estimate: - Load `total_words` and file counts from `graphify-out/.graphify_detect.json` @@ -220,7 +220,7 @@ SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from graphify.cache import check_semantic_cache from pathlib import Path @@ -230,7 +230,7 @@ detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encodin # every source file (#1392). Video is transcribed to a document in Step 2.5 first. all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root=sys.argv[1], prompt_file=sys.argv[2]) # Always (re)write the cache file: write hits, else DELETE any leftover from a prior # run so Part C never merges a stale .graphify_cached.json (#1392). @@ -240,7 +240,7 @@ else: Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" +" "INPUT_PATH" "SPEC_PATH" ``` Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. @@ -282,7 +282,7 @@ Wait for all subagents. For each result: If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: +Merge all chunk files into `.graphify_semantic_new.json`. **After each subagent call completes, read the real token counts from the subagent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: ```bash $(cat graphify-out/.graphify_python) -c " import json, glob @@ -309,15 +309,15 @@ print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens' Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root=sys.argv[1], allowed_source_files=uncached, prompt_file=sys.argv[2]) print(f'Cached {saved} files') -" +" "INPUT_PATH" "SPEC_PATH" ``` Merge cached + new results into `graphify-out/.graphify_semantic.json`: @@ -406,7 +406,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -431,7 +431,7 @@ if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') print('If this shrink is intentional (you deleted files), re-run a full build with --force.') raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, sys.argv[1], suggested_questions=questions) Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") analysis = { 'communities': {str(k): v for k, v in communities.items()}, @@ -442,7 +442,7 @@ analysis = { } Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" +" "INPUT_PATH" ``` If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. @@ -455,12 +455,12 @@ A non-destructive diagnostic on the extraction, before labeling. It surfaces edg ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from pathlib import Path from graphify.diagnostics import diagnose_extraction, format_diagnostic_report extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') +summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root=sys.argv[1]) print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( ('dangling_endpoint_edges', 'dangling-endpoint edges'), @@ -470,7 +470,7 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0)] print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') -" +" "INPUT_PATH" ``` Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules). @@ -496,7 +496,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} @@ -507,7 +507,7 @@ labels = LABELS_DICT # Regenerate questions with real community labels (labels affect question phrasing) questions = suggest_questions(G, communities, labels) -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, sys.argv[1], suggested_questions=questions) # Re-export so graph.json nodes carry the curated community_name (#2490). # Same extraction as Step 4, so the #479 shrink-guard passes on node count; # if it still refuses, surface the guard message - do not force past it. @@ -519,7 +519,7 @@ if not wrote: Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") print('Report updated with community labels') -" +" "INPUT_PATH" ``` Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). @@ -554,7 +554,7 @@ These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, ` ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from pathlib import Path from datetime import datetime, timezone from graphify.detect import save_manifest @@ -577,7 +577,7 @@ extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encod # types are gated on output. from graphify.cli import _stamped_manifest_files _corpus = detect.get('all_files') or detect['files'] -_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) +_manifest_files = _stamped_manifest_files(_corpus, extract, Path(sys.argv[1])) # Files dispatched this run (the changed subset) but NOT stamped above still carry # a stale semantic_hash from a prior run; clear it so detect_incremental re-queues # them instead of reading them as unchanged (#1948). @@ -589,7 +589,7 @@ _cleared = _dispatched - _stamped # files newly excluded since last run are dropped rather than masquerading as # deletions; untouched files' prior rows are still preserved (#1908). _scan = {f for fl in _corpus.values() for f in fl} -save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) # Update cumulative cost tracker input_tok = extract.get('input_tokens', 0) @@ -613,7 +613,7 @@ cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\" print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" +" "INPUT_PATH" rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null rm -f graphify-out/needs_update 2>/dev/null || true diff --git a/tools/skillgen/expected/graphify__skill-kilo.md b/tools/skillgen/expected/graphify__skill-kilo.md index 402fc962e7..05d3e9f4a0 100644 --- a/tools/skillgen/expected/graphify__skill-kilo.md +++ b/tools/skillgen/expected/graphify__skill-kilo.md @@ -108,15 +108,15 @@ If the import succeeds, print nothing and move straight to Step 2. ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from graphify.detect import detect from pathlib import Path -result = detect(Path('INPUT_PATH')) +result = detect(Path(sys.argv[1])) # Write the sidecar from Python, not a shell redirect, so the same block renders # on PowerShell hosts without console-encoding drift (#2528). Path('graphify-out/.graphify_detect.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") print(f'Detected {result[\"total_files\"]} files') -" +" "INPUT_PATH" ``` Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: @@ -183,13 +183,13 @@ for f in detect.get('files', {}).get('code', []): code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) if code_files: - result = extract(code_files, cache_root=Path('INPUT_PATH')) + result = extract(code_files, cache_root=Path(sys.argv[1])) Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') else: Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") print('No code files - skipping AST extraction') -" +" "INPUT_PATH" ``` #### Part B - Semantic extraction (parallel subagents) @@ -204,7 +204,7 @@ Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],' " ``` -**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** +**MANDATORY: You MUST use the subagent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the subagent tool you are doing this wrong.** Before dispatching subagents, print a timing estimate: - Load `total_words` and file counts from `graphify-out/.graphify_detect.json` @@ -220,7 +220,7 @@ SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from graphify.cache import check_semantic_cache from pathlib import Path @@ -230,7 +230,7 @@ detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encodin # every source file (#1392). Video is transcribed to a document in Step 2.5 first. all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root=sys.argv[1], prompt_file=sys.argv[2]) # Always (re)write the cache file: write hits, else DELETE any leftover from a prior # run so Part C never merges a stale .graphify_cached.json (#1392). @@ -240,7 +240,7 @@ else: Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" +" "INPUT_PATH" "SPEC_PATH" ``` Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. @@ -285,7 +285,7 @@ Wait for all subagents. For each result: If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: +Merge all chunk files into `.graphify_semantic_new.json`. **After each subagent call completes, read the real token counts from the subagent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: ```bash $(cat graphify-out/.graphify_python) -c " import json, glob @@ -312,15 +312,15 @@ print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens' Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root=sys.argv[1], allowed_source_files=uncached, prompt_file=sys.argv[2]) print(f'Cached {saved} files') -" +" "INPUT_PATH" "SPEC_PATH" ``` Merge cached + new results into `graphify-out/.graphify_semantic.json`: @@ -409,7 +409,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -434,7 +434,7 @@ if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') print('If this shrink is intentional (you deleted files), re-run a full build with --force.') raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, sys.argv[1], suggested_questions=questions) Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") analysis = { 'communities': {str(k): v for k, v in communities.items()}, @@ -445,7 +445,7 @@ analysis = { } Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" +" "INPUT_PATH" ``` If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. @@ -458,12 +458,12 @@ A non-destructive diagnostic on the extraction, before labeling. It surfaces edg ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from pathlib import Path from graphify.diagnostics import diagnose_extraction, format_diagnostic_report extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') +summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root=sys.argv[1]) print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( ('dangling_endpoint_edges', 'dangling-endpoint edges'), @@ -473,7 +473,7 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0)] print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') -" +" "INPUT_PATH" ``` Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules). @@ -499,7 +499,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} @@ -510,7 +510,7 @@ labels = LABELS_DICT # Regenerate questions with real community labels (labels affect question phrasing) questions = suggest_questions(G, communities, labels) -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, sys.argv[1], suggested_questions=questions) # Re-export so graph.json nodes carry the curated community_name (#2490). # Same extraction as Step 4, so the #479 shrink-guard passes on node count; # if it still refuses, surface the guard message - do not force past it. @@ -522,7 +522,7 @@ if not wrote: Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") print('Report updated with community labels') -" +" "INPUT_PATH" ``` Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). @@ -557,7 +557,7 @@ These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, ` ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from pathlib import Path from datetime import datetime, timezone from graphify.detect import save_manifest @@ -580,7 +580,7 @@ extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encod # types are gated on output. from graphify.cli import _stamped_manifest_files _corpus = detect.get('all_files') or detect['files'] -_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) +_manifest_files = _stamped_manifest_files(_corpus, extract, Path(sys.argv[1])) # Files dispatched this run (the changed subset) but NOT stamped above still carry # a stale semantic_hash from a prior run; clear it so detect_incremental re-queues # them instead of reading them as unchanged (#1948). @@ -592,7 +592,7 @@ _cleared = _dispatched - _stamped # files newly excluded since last run are dropped rather than masquerading as # deletions; untouched files' prior rows are still preserved (#1908). _scan = {f for fl in _corpus.values() for f in fl} -save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) # Update cumulative cost tracker input_tok = extract.get('input_tokens', 0) @@ -616,7 +616,7 @@ cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\" print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" +" "INPUT_PATH" rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null rm -f graphify-out/needs_update 2>/dev/null || true diff --git a/tools/skillgen/expected/graphify__skill-kiro.md b/tools/skillgen/expected/graphify__skill-kiro.md index 56c5c75fc2..7e7c797563 100644 --- a/tools/skillgen/expected/graphify__skill-kiro.md +++ b/tools/skillgen/expected/graphify__skill-kiro.md @@ -108,15 +108,15 @@ If the import succeeds, print nothing and move straight to Step 2. ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from graphify.detect import detect from pathlib import Path -result = detect(Path('INPUT_PATH')) +result = detect(Path(sys.argv[1])) # Write the sidecar from Python, not a shell redirect, so the same block renders # on PowerShell hosts without console-encoding drift (#2528). Path('graphify-out/.graphify_detect.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") print(f'Detected {result[\"total_files\"]} files') -" +" "INPUT_PATH" ``` Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: @@ -183,13 +183,13 @@ for f in detect.get('files', {}).get('code', []): code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) if code_files: - result = extract(code_files, cache_root=Path('INPUT_PATH')) + result = extract(code_files, cache_root=Path(sys.argv[1])) Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') else: Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") print('No code files - skipping AST extraction') -" +" "INPUT_PATH" ``` #### Part B - Semantic extraction (parallel subagents) @@ -204,7 +204,7 @@ Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],' " ``` -**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** +**MANDATORY: You MUST use the subagent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the subagent tool you are doing this wrong.** Before dispatching subagents, print a timing estimate: - Load `total_words` and file counts from `graphify-out/.graphify_detect.json` @@ -220,7 +220,7 @@ SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from graphify.cache import check_semantic_cache from pathlib import Path @@ -230,7 +230,7 @@ detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encodin # every source file (#1392). Video is transcribed to a document in Step 2.5 first. all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root=sys.argv[1], prompt_file=sys.argv[2]) # Always (re)write the cache file: write hits, else DELETE any leftover from a prior # run so Part C never merges a stale .graphify_cached.json (#1392). @@ -240,7 +240,7 @@ else: Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" +" "INPUT_PATH" "SPEC_PATH" ``` Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. @@ -285,7 +285,7 @@ Wait for all subagents. For each result: If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: +Merge all chunk files into `.graphify_semantic_new.json`. **After each subagent call completes, read the real token counts from the subagent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: ```bash $(cat graphify-out/.graphify_python) -c " import json, glob @@ -312,15 +312,15 @@ print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens' Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root=sys.argv[1], allowed_source_files=uncached, prompt_file=sys.argv[2]) print(f'Cached {saved} files') -" +" "INPUT_PATH" "SPEC_PATH" ``` Merge cached + new results into `graphify-out/.graphify_semantic.json`: @@ -409,7 +409,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -434,7 +434,7 @@ if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') print('If this shrink is intentional (you deleted files), re-run a full build with --force.') raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, sys.argv[1], suggested_questions=questions) Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") analysis = { 'communities': {str(k): v for k, v in communities.items()}, @@ -445,7 +445,7 @@ analysis = { } Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" +" "INPUT_PATH" ``` If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. @@ -458,12 +458,12 @@ A non-destructive diagnostic on the extraction, before labeling. It surfaces edg ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from pathlib import Path from graphify.diagnostics import diagnose_extraction, format_diagnostic_report extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') +summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root=sys.argv[1]) print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( ('dangling_endpoint_edges', 'dangling-endpoint edges'), @@ -473,7 +473,7 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0)] print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') -" +" "INPUT_PATH" ``` Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules). @@ -499,7 +499,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} @@ -510,7 +510,7 @@ labels = LABELS_DICT # Regenerate questions with real community labels (labels affect question phrasing) questions = suggest_questions(G, communities, labels) -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, sys.argv[1], suggested_questions=questions) # Re-export so graph.json nodes carry the curated community_name (#2490). # Same extraction as Step 4, so the #479 shrink-guard passes on node count; # if it still refuses, surface the guard message - do not force past it. @@ -522,7 +522,7 @@ if not wrote: Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") print('Report updated with community labels') -" +" "INPUT_PATH" ``` Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). @@ -557,7 +557,7 @@ These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, ` ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from pathlib import Path from datetime import datetime, timezone from graphify.detect import save_manifest @@ -580,7 +580,7 @@ extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encod # types are gated on output. from graphify.cli import _stamped_manifest_files _corpus = detect.get('all_files') or detect['files'] -_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) +_manifest_files = _stamped_manifest_files(_corpus, extract, Path(sys.argv[1])) # Files dispatched this run (the changed subset) but NOT stamped above still carry # a stale semantic_hash from a prior run; clear it so detect_incremental re-queues # them instead of reading them as unchanged (#1948). @@ -592,7 +592,7 @@ _cleared = _dispatched - _stamped # files newly excluded since last run are dropped rather than masquerading as # deletions; untouched files' prior rows are still preserved (#1908). _scan = {f for fl in _corpus.values() for f in fl} -save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) # Update cumulative cost tracker input_tok = extract.get('input_tokens', 0) @@ -616,7 +616,7 @@ cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\" print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" +" "INPUT_PATH" rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null rm -f graphify-out/needs_update 2>/dev/null || true diff --git a/tools/skillgen/expected/graphify__skill-opencode.md b/tools/skillgen/expected/graphify__skill-opencode.md index f8e24e6272..313958ae2d 100644 --- a/tools/skillgen/expected/graphify__skill-opencode.md +++ b/tools/skillgen/expected/graphify__skill-opencode.md @@ -108,15 +108,15 @@ If the import succeeds, print nothing and move straight to Step 2. ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from graphify.detect import detect from pathlib import Path -result = detect(Path('INPUT_PATH')) +result = detect(Path(sys.argv[1])) # Write the sidecar from Python, not a shell redirect, so the same block renders # on PowerShell hosts without console-encoding drift (#2528). Path('graphify-out/.graphify_detect.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") print(f'Detected {result[\"total_files\"]} files') -" +" "INPUT_PATH" ``` Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: @@ -183,13 +183,13 @@ for f in detect.get('files', {}).get('code', []): code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) if code_files: - result = extract(code_files, cache_root=Path('INPUT_PATH')) + result = extract(code_files, cache_root=Path(sys.argv[1])) Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') else: Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") print('No code files - skipping AST extraction') -" +" "INPUT_PATH" ``` #### Part B - Semantic extraction (parallel subagents) @@ -204,7 +204,7 @@ Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],' " ``` -**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** +**MANDATORY: You MUST use the subagent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the subagent tool you are doing this wrong.** Before dispatching subagents, print a timing estimate: - Load `total_words` and file counts from `graphify-out/.graphify_detect.json` @@ -220,7 +220,7 @@ SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from graphify.cache import check_semantic_cache from pathlib import Path @@ -230,7 +230,7 @@ detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encodin # every source file (#1392). Video is transcribed to a document in Step 2.5 first. all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root=sys.argv[1], prompt_file=sys.argv[2]) # Always (re)write the cache file: write hits, else DELETE any leftover from a prior # run so Part C never merges a stale .graphify_cached.json (#1392). @@ -240,7 +240,7 @@ else: Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" +" "INPUT_PATH" "SPEC_PATH" ``` Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. @@ -277,7 +277,7 @@ Wait for all subagents. For each result: If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: +Merge all chunk files into `.graphify_semantic_new.json`. **After each subagent call completes, read the real token counts from the subagent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: ```bash $(cat graphify-out/.graphify_python) -c " import json, glob @@ -304,15 +304,15 @@ print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens' Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root=sys.argv[1], allowed_source_files=uncached, prompt_file=sys.argv[2]) print(f'Cached {saved} files') -" +" "INPUT_PATH" "SPEC_PATH" ``` Merge cached + new results into `graphify-out/.graphify_semantic.json`: @@ -401,7 +401,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -426,7 +426,7 @@ if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') print('If this shrink is intentional (you deleted files), re-run a full build with --force.') raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, sys.argv[1], suggested_questions=questions) Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") analysis = { 'communities': {str(k): v for k, v in communities.items()}, @@ -437,7 +437,7 @@ analysis = { } Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" +" "INPUT_PATH" ``` If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. @@ -450,12 +450,12 @@ A non-destructive diagnostic on the extraction, before labeling. It surfaces edg ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from pathlib import Path from graphify.diagnostics import diagnose_extraction, format_diagnostic_report extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') +summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root=sys.argv[1]) print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( ('dangling_endpoint_edges', 'dangling-endpoint edges'), @@ -465,7 +465,7 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0)] print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') -" +" "INPUT_PATH" ``` Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules). @@ -491,7 +491,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} @@ -502,7 +502,7 @@ labels = LABELS_DICT # Regenerate questions with real community labels (labels affect question phrasing) questions = suggest_questions(G, communities, labels) -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, sys.argv[1], suggested_questions=questions) # Re-export so graph.json nodes carry the curated community_name (#2490). # Same extraction as Step 4, so the #479 shrink-guard passes on node count; # if it still refuses, surface the guard message - do not force past it. @@ -514,7 +514,7 @@ if not wrote: Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") print('Report updated with community labels') -" +" "INPUT_PATH" ``` Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). @@ -549,7 +549,7 @@ These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, ` ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from pathlib import Path from datetime import datetime, timezone from graphify.detect import save_manifest @@ -572,7 +572,7 @@ extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encod # types are gated on output. from graphify.cli import _stamped_manifest_files _corpus = detect.get('all_files') or detect['files'] -_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) +_manifest_files = _stamped_manifest_files(_corpus, extract, Path(sys.argv[1])) # Files dispatched this run (the changed subset) but NOT stamped above still carry # a stale semantic_hash from a prior run; clear it so detect_incremental re-queues # them instead of reading them as unchanged (#1948). @@ -584,7 +584,7 @@ _cleared = _dispatched - _stamped # files newly excluded since last run are dropped rather than masquerading as # deletions; untouched files' prior rows are still preserved (#1908). _scan = {f for fl in _corpus.values() for f in fl} -save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) # Update cumulative cost tracker input_tok = extract.get('input_tokens', 0) @@ -608,7 +608,7 @@ cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\" print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" +" "INPUT_PATH" rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null rm -f graphify-out/needs_update 2>/dev/null || true diff --git a/tools/skillgen/expected/graphify__skill-pi.md b/tools/skillgen/expected/graphify__skill-pi.md index 56c5c75fc2..7e7c797563 100644 --- a/tools/skillgen/expected/graphify__skill-pi.md +++ b/tools/skillgen/expected/graphify__skill-pi.md @@ -108,15 +108,15 @@ If the import succeeds, print nothing and move straight to Step 2. ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from graphify.detect import detect from pathlib import Path -result = detect(Path('INPUT_PATH')) +result = detect(Path(sys.argv[1])) # Write the sidecar from Python, not a shell redirect, so the same block renders # on PowerShell hosts without console-encoding drift (#2528). Path('graphify-out/.graphify_detect.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") print(f'Detected {result[\"total_files\"]} files') -" +" "INPUT_PATH" ``` Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: @@ -183,13 +183,13 @@ for f in detect.get('files', {}).get('code', []): code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) if code_files: - result = extract(code_files, cache_root=Path('INPUT_PATH')) + result = extract(code_files, cache_root=Path(sys.argv[1])) Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') else: Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") print('No code files - skipping AST extraction') -" +" "INPUT_PATH" ``` #### Part B - Semantic extraction (parallel subagents) @@ -204,7 +204,7 @@ Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],' " ``` -**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** +**MANDATORY: You MUST use the subagent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the subagent tool you are doing this wrong.** Before dispatching subagents, print a timing estimate: - Load `total_words` and file counts from `graphify-out/.graphify_detect.json` @@ -220,7 +220,7 @@ SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from graphify.cache import check_semantic_cache from pathlib import Path @@ -230,7 +230,7 @@ detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encodin # every source file (#1392). Video is transcribed to a document in Step 2.5 first. all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root=sys.argv[1], prompt_file=sys.argv[2]) # Always (re)write the cache file: write hits, else DELETE any leftover from a prior # run so Part C never merges a stale .graphify_cached.json (#1392). @@ -240,7 +240,7 @@ else: Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" +" "INPUT_PATH" "SPEC_PATH" ``` Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. @@ -285,7 +285,7 @@ Wait for all subagents. For each result: If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: +Merge all chunk files into `.graphify_semantic_new.json`. **After each subagent call completes, read the real token counts from the subagent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: ```bash $(cat graphify-out/.graphify_python) -c " import json, glob @@ -312,15 +312,15 @@ print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens' Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root=sys.argv[1], allowed_source_files=uncached, prompt_file=sys.argv[2]) print(f'Cached {saved} files') -" +" "INPUT_PATH" "SPEC_PATH" ``` Merge cached + new results into `graphify-out/.graphify_semantic.json`: @@ -409,7 +409,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -434,7 +434,7 @@ if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') print('If this shrink is intentional (you deleted files), re-run a full build with --force.') raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, sys.argv[1], suggested_questions=questions) Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") analysis = { 'communities': {str(k): v for k, v in communities.items()}, @@ -445,7 +445,7 @@ analysis = { } Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" +" "INPUT_PATH" ``` If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. @@ -458,12 +458,12 @@ A non-destructive diagnostic on the extraction, before labeling. It surfaces edg ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from pathlib import Path from graphify.diagnostics import diagnose_extraction, format_diagnostic_report extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') +summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root=sys.argv[1]) print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( ('dangling_endpoint_edges', 'dangling-endpoint edges'), @@ -473,7 +473,7 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0)] print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') -" +" "INPUT_PATH" ``` Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules). @@ -499,7 +499,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} @@ -510,7 +510,7 @@ labels = LABELS_DICT # Regenerate questions with real community labels (labels affect question phrasing) questions = suggest_questions(G, communities, labels) -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, sys.argv[1], suggested_questions=questions) # Re-export so graph.json nodes carry the curated community_name (#2490). # Same extraction as Step 4, so the #479 shrink-guard passes on node count; # if it still refuses, surface the guard message - do not force past it. @@ -522,7 +522,7 @@ if not wrote: Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") print('Report updated with community labels') -" +" "INPUT_PATH" ``` Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). @@ -557,7 +557,7 @@ These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, ` ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from pathlib import Path from datetime import datetime, timezone from graphify.detect import save_manifest @@ -580,7 +580,7 @@ extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encod # types are gated on output. from graphify.cli import _stamped_manifest_files _corpus = detect.get('all_files') or detect['files'] -_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) +_manifest_files = _stamped_manifest_files(_corpus, extract, Path(sys.argv[1])) # Files dispatched this run (the changed subset) but NOT stamped above still carry # a stale semantic_hash from a prior run; clear it so detect_incremental re-queues # them instead of reading them as unchanged (#1948). @@ -592,7 +592,7 @@ _cleared = _dispatched - _stamped # files newly excluded since last run are dropped rather than masquerading as # deletions; untouched files' prior rows are still preserved (#1908). _scan = {f for fl in _corpus.values() for f in fl} -save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) # Update cumulative cost tracker input_tok = extract.get('input_tokens', 0) @@ -616,7 +616,7 @@ cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\" print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" +" "INPUT_PATH" rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null rm -f graphify-out/needs_update 2>/dev/null || true diff --git a/tools/skillgen/expected/graphify__skill-trae.md b/tools/skillgen/expected/graphify__skill-trae.md index 55f8c2caca..2b73e78216 100644 --- a/tools/skillgen/expected/graphify__skill-trae.md +++ b/tools/skillgen/expected/graphify__skill-trae.md @@ -108,15 +108,15 @@ If the import succeeds, print nothing and move straight to Step 2. ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from graphify.detect import detect from pathlib import Path -result = detect(Path('INPUT_PATH')) +result = detect(Path(sys.argv[1])) # Write the sidecar from Python, not a shell redirect, so the same block renders # on PowerShell hosts without console-encoding drift (#2528). Path('graphify-out/.graphify_detect.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") print(f'Detected {result[\"total_files\"]} files') -" +" "INPUT_PATH" ``` Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: @@ -183,13 +183,13 @@ for f in detect.get('files', {}).get('code', []): code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) if code_files: - result = extract(code_files, cache_root=Path('INPUT_PATH')) + result = extract(code_files, cache_root=Path(sys.argv[1])) Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') else: Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") print('No code files - skipping AST extraction') -" +" "INPUT_PATH" ``` #### Part B - Semantic extraction (parallel subagents) @@ -204,7 +204,7 @@ Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],' " ``` -**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** +**MANDATORY: You MUST use the subagent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the subagent tool you are doing this wrong.** Before dispatching subagents, print a timing estimate: - Load `total_words` and file counts from `graphify-out/.graphify_detect.json` @@ -220,7 +220,7 @@ SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from graphify.cache import check_semantic_cache from pathlib import Path @@ -230,7 +230,7 @@ detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encodin # every source file (#1392). Video is transcribed to a document in Step 2.5 first. all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root=sys.argv[1], prompt_file=sys.argv[2]) # Always (re)write the cache file: write hits, else DELETE any leftover from a prior # run so Part C never merges a stale .graphify_cached.json (#1392). @@ -240,7 +240,7 @@ else: Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" +" "INPUT_PATH" "SPEC_PATH" ``` Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. @@ -283,7 +283,7 @@ Wait for all subagents. For each result: If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: +Merge all chunk files into `.graphify_semantic_new.json`. **After each subagent call completes, read the real token counts from the subagent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: ```bash $(cat graphify-out/.graphify_python) -c " import json, glob @@ -310,15 +310,15 @@ print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens' Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root=sys.argv[1], allowed_source_files=uncached, prompt_file=sys.argv[2]) print(f'Cached {saved} files') -" +" "INPUT_PATH" "SPEC_PATH" ``` Merge cached + new results into `graphify-out/.graphify_semantic.json`: @@ -407,7 +407,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -432,7 +432,7 @@ if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') print('If this shrink is intentional (you deleted files), re-run a full build with --force.') raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, sys.argv[1], suggested_questions=questions) Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") analysis = { 'communities': {str(k): v for k, v in communities.items()}, @@ -443,7 +443,7 @@ analysis = { } Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" +" "INPUT_PATH" ``` If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. @@ -456,12 +456,12 @@ A non-destructive diagnostic on the extraction, before labeling. It surfaces edg ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from pathlib import Path from graphify.diagnostics import diagnose_extraction, format_diagnostic_report extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') +summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root=sys.argv[1]) print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( ('dangling_endpoint_edges', 'dangling-endpoint edges'), @@ -471,7 +471,7 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0)] print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') -" +" "INPUT_PATH" ``` Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules). @@ -497,7 +497,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} @@ -508,7 +508,7 @@ labels = LABELS_DICT # Regenerate questions with real community labels (labels affect question phrasing) questions = suggest_questions(G, communities, labels) -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, sys.argv[1], suggested_questions=questions) # Re-export so graph.json nodes carry the curated community_name (#2490). # Same extraction as Step 4, so the #479 shrink-guard passes on node count; # if it still refuses, surface the guard message - do not force past it. @@ -520,7 +520,7 @@ if not wrote: Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") print('Report updated with community labels') -" +" "INPUT_PATH" ``` Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). @@ -555,7 +555,7 @@ These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, ` ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from pathlib import Path from datetime import datetime, timezone from graphify.detect import save_manifest @@ -578,7 +578,7 @@ extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encod # types are gated on output. from graphify.cli import _stamped_manifest_files _corpus = detect.get('all_files') or detect['files'] -_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) +_manifest_files = _stamped_manifest_files(_corpus, extract, Path(sys.argv[1])) # Files dispatched this run (the changed subset) but NOT stamped above still carry # a stale semantic_hash from a prior run; clear it so detect_incremental re-queues # them instead of reading them as unchanged (#1948). @@ -590,7 +590,7 @@ _cleared = _dispatched - _stamped # files newly excluded since last run are dropped rather than masquerading as # deletions; untouched files' prior rows are still preserved (#1908). _scan = {f for fl in _corpus.values() for f in fl} -save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) # Update cumulative cost tracker input_tok = extract.get('input_tokens', 0) @@ -614,7 +614,7 @@ cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\" print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" +" "INPUT_PATH" rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null rm -f graphify-out/needs_update 2>/dev/null || true diff --git a/tools/skillgen/expected/graphify__skill-vscode.md b/tools/skillgen/expected/graphify__skill-vscode.md index 9151a264b2..c187cd7d33 100644 --- a/tools/skillgen/expected/graphify__skill-vscode.md +++ b/tools/skillgen/expected/graphify__skill-vscode.md @@ -108,15 +108,15 @@ If the import succeeds, print nothing and move straight to Step 2. ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from graphify.detect import detect from pathlib import Path -result = detect(Path('INPUT_PATH')) +result = detect(Path(sys.argv[1])) # Write the sidecar from Python, not a shell redirect, so the same block renders # on PowerShell hosts without console-encoding drift (#2528). Path('graphify-out/.graphify_detect.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") print(f'Detected {result[\"total_files\"]} files') -" +" "INPUT_PATH" ``` Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: @@ -183,13 +183,13 @@ for f in detect.get('files', {}).get('code', []): code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) if code_files: - result = extract(code_files, cache_root=Path('INPUT_PATH')) + result = extract(code_files, cache_root=Path(sys.argv[1])) Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') else: Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") print('No code files - skipping AST extraction') -" +" "INPUT_PATH" ``` #### Part B - Semantic extraction (parallel subagents) @@ -204,7 +204,7 @@ Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],' " ``` -**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** +**MANDATORY: You MUST use the subagent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the subagent tool you are doing this wrong.** Before dispatching subagents, print a timing estimate: - Load `total_words` and file counts from `graphify-out/.graphify_detect.json` @@ -220,7 +220,7 @@ SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from graphify.cache import check_semantic_cache from pathlib import Path @@ -230,7 +230,7 @@ detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encodin # every source file (#1392). Video is transcribed to a document in Step 2.5 first. all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root=sys.argv[1], prompt_file=sys.argv[2]) # Always (re)write the cache file: write hits, else DELETE any leftover from a prior # run so Part C never merges a stale .graphify_cached.json (#1392). @@ -240,7 +240,7 @@ else: Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" +" "INPUT_PATH" "SPEC_PATH" ``` Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. @@ -281,7 +281,7 @@ Wait for all subagents. For each result: If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: +Merge all chunk files into `.graphify_semantic_new.json`. **After each subagent call completes, read the real token counts from the subagent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: ```bash $(cat graphify-out/.graphify_python) -c " import json, glob @@ -308,15 +308,15 @@ print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens' Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root=sys.argv[1], allowed_source_files=uncached, prompt_file=sys.argv[2]) print(f'Cached {saved} files') -" +" "INPUT_PATH" "SPEC_PATH" ``` Merge cached + new results into `graphify-out/.graphify_semantic.json`: @@ -405,7 +405,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -430,7 +430,7 @@ if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') print('If this shrink is intentional (you deleted files), re-run a full build with --force.') raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, sys.argv[1], suggested_questions=questions) Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") analysis = { 'communities': {str(k): v for k, v in communities.items()}, @@ -441,7 +441,7 @@ analysis = { } Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" +" "INPUT_PATH" ``` If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. @@ -454,12 +454,12 @@ A non-destructive diagnostic on the extraction, before labeling. It surfaces edg ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from pathlib import Path from graphify.diagnostics import diagnose_extraction, format_diagnostic_report extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') +summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root=sys.argv[1]) print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( ('dangling_endpoint_edges', 'dangling-endpoint edges'), @@ -469,7 +469,7 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0)] print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') -" +" "INPUT_PATH" ``` Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules). @@ -495,7 +495,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} @@ -506,7 +506,7 @@ labels = LABELS_DICT # Regenerate questions with real community labels (labels affect question phrasing) questions = suggest_questions(G, communities, labels) -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, sys.argv[1], suggested_questions=questions) # Re-export so graph.json nodes carry the curated community_name (#2490). # Same extraction as Step 4, so the #479 shrink-guard passes on node count; # if it still refuses, surface the guard message - do not force past it. @@ -518,7 +518,7 @@ if not wrote: Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") print('Report updated with community labels') -" +" "INPUT_PATH" ``` Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). @@ -553,7 +553,7 @@ These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, ` ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from pathlib import Path from datetime import datetime, timezone from graphify.detect import save_manifest @@ -576,7 +576,7 @@ extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encod # types are gated on output. from graphify.cli import _stamped_manifest_files _corpus = detect.get('all_files') or detect['files'] -_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) +_manifest_files = _stamped_manifest_files(_corpus, extract, Path(sys.argv[1])) # Files dispatched this run (the changed subset) but NOT stamped above still carry # a stale semantic_hash from a prior run; clear it so detect_incremental re-queues # them instead of reading them as unchanged (#1948). @@ -588,7 +588,7 @@ _cleared = _dispatched - _stamped # files newly excluded since last run are dropped rather than masquerading as # deletions; untouched files' prior rows are still preserved (#1908). _scan = {f for fl in _corpus.values() for f in fl} -save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) # Update cumulative cost tracker input_tok = extract.get('input_tokens', 0) @@ -612,7 +612,7 @@ cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\" print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" +" "INPUT_PATH" rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null rm -f graphify-out/needs_update 2>/dev/null || true diff --git a/tools/skillgen/expected/graphify__skill-windows.md b/tools/skillgen/expected/graphify__skill-windows.md index 81dc1b133c..9534fbd706 100644 --- a/tools/skillgen/expected/graphify__skill-windows.md +++ b/tools/skillgen/expected/graphify__skill-windows.md @@ -135,15 +135,15 @@ If the import succeeds, print nothing and move straight to Step 2. ```powershell @' -import json +import sys, json from graphify.detect import detect from pathlib import Path -result = detect(Path('INPUT_PATH')) +result = detect(Path(sys.argv[1])) # Write the sidecar from Python, not a shell redirect, so the same block renders # on PowerShell hosts without console-encoding drift (#2528). Path('graphify-out/.graphify_detect.json').write_text(json.dumps(result, ensure_ascii=False), encoding="utf-8") print(f'Detected {result["total_files"]} files') -'@ | & (Get-Content graphify-out\.graphify_python) - +'@ | & (Get-Content graphify-out\.graphify_python) - "INPUT_PATH" ``` Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: @@ -210,13 +210,13 @@ for f in detect.get('files', {}).get('code', []): code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) if code_files: - result = extract(code_files, cache_root=Path('INPUT_PATH')) + result = extract(code_files, cache_root=Path(sys.argv[1])) Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding="utf-8") print(f'AST: {len(result["nodes"])} nodes, {len(result["edges"])} edges') else: Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding="utf-8") print('No code files - skipping AST extraction') -'@ | & (Get-Content graphify-out\.graphify_python) - +'@ | & (Get-Content graphify-out\.graphify_python) - "INPUT_PATH" ``` #### Part B - Semantic extraction (parallel subagents) @@ -231,7 +231,7 @@ Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],' '@ | & (Get-Content graphify-out\.graphify_python) - ``` -**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** +**MANDATORY: You MUST use the subagent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the subagent tool you are doing this wrong.** Before dispatching subagents, print a timing estimate: - Load `total_words` and file counts from `graphify-out/.graphify_detect.json` @@ -247,7 +247,7 @@ SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` ```powershell @' -import json +import sys, json from graphify.cache import check_semantic_cache from pathlib import Path @@ -257,7 +257,7 @@ detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encodin # every source file (#1392). Video is transcribed to a document in Step 2.5 first. all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root=sys.argv[1], prompt_file=sys.argv[2]) # Always (re)write the cache file: write hits, else DELETE any leftover from a prior # run so Part C never merges a stale .graphify_cached.json (#1392). @@ -267,7 +267,7 @@ else: Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding="utf-8") print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -'@ | & (Get-Content graphify-out\.graphify_python) - +'@ | & (Get-Content graphify-out\.graphify_python) - "INPUT_PATH" "SPEC_PATH" ``` Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. @@ -312,7 +312,7 @@ Wait for all subagents. For each result: If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: +Merge all chunk files into `.graphify_semantic_new.json`. **After each subagent call completes, read the real token counts from the subagent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: ```powershell @' import json, glob @@ -339,15 +339,15 @@ print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens' Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): ```powershell @' -import json +import sys, json from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding="utf-8")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding="utf-8").splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root=sys.argv[1], allowed_source_files=uncached, prompt_file=sys.argv[2]) print(f'Cached {saved} files') -'@ | & (Get-Content graphify-out\.graphify_python) - +'@ | & (Get-Content graphify-out\.graphify_python) - "INPUT_PATH" "SPEC_PATH" ``` Merge cached + new results into `graphify-out/.graphify_semantic.json`: @@ -436,7 +436,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -461,7 +461,7 @@ if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') print('If this shrink is intentional (you deleted files), re-run a full build with --force.') raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, sys.argv[1], suggested_questions=questions) Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding="utf-8") analysis = { 'communities': {str(k): v for k, v in communities.items()}, @@ -472,7 +472,7 @@ analysis = { } Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding="utf-8") print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -'@ | & (Get-Content graphify-out\.graphify_python) - +'@ | & (Get-Content graphify-out\.graphify_python) - "INPUT_PATH" ``` If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. @@ -485,12 +485,12 @@ A non-destructive diagnostic on the extraction, before labeling. It surfaces edg ```powershell @' -import json +import sys, json from pathlib import Path from graphify.diagnostics import diagnose_extraction, format_diagnostic_report extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding="utf-8")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') +summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root=sys.argv[1]) print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( ('dangling_endpoint_edges', 'dangling-endpoint edges'), @@ -500,7 +500,7 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0)] print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') -'@ | & (Get-Content graphify-out\.graphify_python) - +'@ | & (Get-Content graphify-out\.graphify_python) - "INPUT_PATH" ``` Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules). @@ -526,7 +526,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding="utf-8")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} @@ -537,7 +537,7 @@ labels = LABELS_DICT # Regenerate questions with real community labels (labels affect question phrasing) questions = suggest_questions(G, communities, labels) -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, sys.argv[1], suggested_questions=questions) # Re-export so graph.json nodes carry the curated community_name (#2490). # Same extraction as Step 4, so the #479 shrink-guard passes on node count; # if it still refuses, surface the guard message - do not force past it. @@ -549,7 +549,7 @@ if not wrote: Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding="utf-8") Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding="utf-8") print('Report updated with community labels') -'@ | & (Get-Content graphify-out\.graphify_python) - +'@ | & (Get-Content graphify-out\.graphify_python) - "INPUT_PATH" ``` Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). @@ -584,7 +584,7 @@ These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, ` ```powershell @' -import json +import sys, json from pathlib import Path from datetime import datetime, timezone from graphify.detect import save_manifest @@ -607,7 +607,7 @@ extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encod # types are gated on output. from graphify.cli import _stamped_manifest_files _corpus = detect.get('all_files') or detect['files'] -_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) +_manifest_files = _stamped_manifest_files(_corpus, extract, Path(sys.argv[1])) # Files dispatched this run (the changed subset) but NOT stamped above still carry # a stale semantic_hash from a prior run; clear it so detect_incremental re-queues # them instead of reading them as unchanged (#1948). @@ -619,7 +619,7 @@ _cleared = _dispatched - _stamped # files newly excluded since last run are dropped rather than masquerading as # deletions; untouched files' prior rows are still preserved (#1908). _scan = {f for fl in _corpus.values() for f in fl} -save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) # Update cumulative cost tracker input_tok = extract.get('input_tokens', 0) @@ -643,7 +643,7 @@ cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding="u print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') print(f'All time: {cost["total_input_tokens"]:,} input, {cost["total_output_tokens"]:,} output ({len(cost["runs"])} runs)') -'@ | & (Get-Content graphify-out\.graphify_python) - +'@ | & (Get-Content graphify-out\.graphify_python) - "INPUT_PATH" Remove-Item -Force -ErrorAction SilentlyContinue graphify-out\.graphify_detect.json, graphify-out\.graphify_extract.json, graphify-out\.graphify_ast.json, graphify-out\.graphify_semantic.json, graphify-out\.graphify_analysis.json Get-ChildItem graphify-out -Filter '.graphify_chunk_*.json' -File -ErrorAction SilentlyContinue | Remove-Item -Force Remove-Item -Force -ErrorAction SilentlyContinue graphify-out\needs_update diff --git a/tools/skillgen/expected/graphify__skill.md b/tools/skillgen/expected/graphify__skill.md index 56c5c75fc2..7e7c797563 100644 --- a/tools/skillgen/expected/graphify__skill.md +++ b/tools/skillgen/expected/graphify__skill.md @@ -108,15 +108,15 @@ If the import succeeds, print nothing and move straight to Step 2. ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from graphify.detect import detect from pathlib import Path -result = detect(Path('INPUT_PATH')) +result = detect(Path(sys.argv[1])) # Write the sidecar from Python, not a shell redirect, so the same block renders # on PowerShell hosts without console-encoding drift (#2528). Path('graphify-out/.graphify_detect.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") print(f'Detected {result[\"total_files\"]} files') -" +" "INPUT_PATH" ``` Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: @@ -183,13 +183,13 @@ for f in detect.get('files', {}).get('code', []): code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) if code_files: - result = extract(code_files, cache_root=Path('INPUT_PATH')) + result = extract(code_files, cache_root=Path(sys.argv[1])) Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') else: Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") print('No code files - skipping AST extraction') -" +" "INPUT_PATH" ``` #### Part B - Semantic extraction (parallel subagents) @@ -204,7 +204,7 @@ Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],' " ``` -**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** +**MANDATORY: You MUST use the subagent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the subagent tool you are doing this wrong.** Before dispatching subagents, print a timing estimate: - Load `total_words` and file counts from `graphify-out/.graphify_detect.json` @@ -220,7 +220,7 @@ SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from graphify.cache import check_semantic_cache from pathlib import Path @@ -230,7 +230,7 @@ detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encodin # every source file (#1392). Video is transcribed to a document in Step 2.5 first. all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root=sys.argv[1], prompt_file=sys.argv[2]) # Always (re)write the cache file: write hits, else DELETE any leftover from a prior # run so Part C never merges a stale .graphify_cached.json (#1392). @@ -240,7 +240,7 @@ else: Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" +" "INPUT_PATH" "SPEC_PATH" ``` Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. @@ -285,7 +285,7 @@ Wait for all subagents. For each result: If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: +Merge all chunk files into `.graphify_semantic_new.json`. **After each subagent call completes, read the real token counts from the subagent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: ```bash $(cat graphify-out/.graphify_python) -c " import json, glob @@ -312,15 +312,15 @@ print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens' Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root=sys.argv[1], allowed_source_files=uncached, prompt_file=sys.argv[2]) print(f'Cached {saved} files') -" +" "INPUT_PATH" "SPEC_PATH" ``` Merge cached + new results into `graphify-out/.graphify_semantic.json`: @@ -409,7 +409,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -434,7 +434,7 @@ if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') print('If this shrink is intentional (you deleted files), re-run a full build with --force.') raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, sys.argv[1], suggested_questions=questions) Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") analysis = { 'communities': {str(k): v for k, v in communities.items()}, @@ -445,7 +445,7 @@ analysis = { } Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" +" "INPUT_PATH" ``` If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. @@ -458,12 +458,12 @@ A non-destructive diagnostic on the extraction, before labeling. It surfaces edg ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from pathlib import Path from graphify.diagnostics import diagnose_extraction, format_diagnostic_report extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') +summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root=sys.argv[1]) print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( ('dangling_endpoint_edges', 'dangling-endpoint edges'), @@ -473,7 +473,7 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0)] print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') -" +" "INPUT_PATH" ``` Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules). @@ -499,7 +499,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} @@ -510,7 +510,7 @@ labels = LABELS_DICT # Regenerate questions with real community labels (labels affect question phrasing) questions = suggest_questions(G, communities, labels) -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, sys.argv[1], suggested_questions=questions) # Re-export so graph.json nodes carry the curated community_name (#2490). # Same extraction as Step 4, so the #479 shrink-guard passes on node count; # if it still refuses, surface the guard message - do not force past it. @@ -522,7 +522,7 @@ if not wrote: Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") print('Report updated with community labels') -" +" "INPUT_PATH" ``` Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). @@ -557,7 +557,7 @@ These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, ` ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from pathlib import Path from datetime import datetime, timezone from graphify.detect import save_manifest @@ -580,7 +580,7 @@ extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encod # types are gated on output. from graphify.cli import _stamped_manifest_files _corpus = detect.get('all_files') or detect['files'] -_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) +_manifest_files = _stamped_manifest_files(_corpus, extract, Path(sys.argv[1])) # Files dispatched this run (the changed subset) but NOT stamped above still carry # a stale semantic_hash from a prior run; clear it so detect_incremental re-queues # them instead of reading them as unchanged (#1948). @@ -592,7 +592,7 @@ _cleared = _dispatched - _stamped # files newly excluded since last run are dropped rather than masquerading as # deletions; untouched files' prior rows are still preserved (#1908). _scan = {f for fl in _corpus.values() for f in fl} -save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) # Update cumulative cost tracker input_tok = extract.get('input_tokens', 0) @@ -616,7 +616,7 @@ cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\" print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" +" "INPUT_PATH" rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null rm -f graphify-out/needs_update 2>/dev/null || true diff --git a/tools/skillgen/expected/graphify__skills__agents__references__transcribe.md b/tools/skillgen/expected/graphify__skills__agents__references__transcribe.md index b967f83799..bbe8b93e45 100644 --- a/tools/skillgen/expected/graphify__skills__agents__references__transcribe.md +++ b/tools/skillgen/expected/graphify__skills__agents__references__transcribe.md @@ -8,13 +8,11 @@ Skip this step entirely if `detect` returned zero `video` files. Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. -**Strategy:** Read the god nodes from `graphify-out/.graphify_detect.json` (or the analysis file if it exists from a previous run). You are already a language model — write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. - -**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` +**Strategy:** If `graphify-out/.graphify_analysis.json` exists from a previous run, read its top god-node labels and write a one-sentence domain hint from them. Otherwise use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` Pass the chosen prompt to Whisper as its initial prompt. No separate API call needed. **Step 1 - Write the Whisper prompt yourself.** -Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: +Read the top god-node labels from previous analysis when available, then compose a short domain hint sentence. With no previous analysis, use the generic fallback above. - Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` - Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` diff --git a/tools/skillgen/expected/graphify__skills__agents__references__update.md b/tools/skillgen/expected/graphify__skills__agents__references__update.md index bd8e659d3c..058ca8e6d2 100644 --- a/tools/skillgen/expected/graphify__skills__agents__references__update.md +++ b/tools/skillgen/expected/graphify__skills__agents__references__update.md @@ -12,7 +12,7 @@ import sys, json from graphify.detect import detect_incremental, save_manifest from pathlib import Path -result = detect_incremental(Path('INPUT_PATH')) +result = detect_incremental(Path(sys.argv[1])) new_total = result.get('new_total', 0) print(json.dumps(result, indent=2, ensure_ascii=False)) Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") @@ -24,7 +24,7 @@ if deleted: print(f'{len(deleted)} deleted file(s) to prune.') if new_total > 0: print(f'{new_total} new/changed file(s) to re-extract.') -" +" "INPUT_PATH" ``` Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: @@ -84,7 +84,7 @@ Then: ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from pathlib import Path from graphify.build import build_merge from graphify.detect import save_manifest @@ -113,7 +113,7 @@ G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, - root='INPUT_PATH', + root=sys.argv[1], directed=IS_DIRECTED, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -150,7 +150,7 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) # is lost forever (#2015). Mirrors the library extract path # (cli._stamped_manifest_files + clear_semantic + scan_corpus). from graphify.cli import _stamped_manifest_files -_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) +_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path(sys.argv[1])) # Changed semantic files dispatched this run but NOT stamped had their chunk fail # or be omitted; clear any stale semantic_hash so they are re-queued (#1948). _sem_types = ('document', 'paper', 'image') @@ -160,9 +160,9 @@ _cleared = _dispatched - _stamped # scan_corpus = the RAW full corpus so in-root files newly excluded since last run # are dropped rather than masquerading as deletions; untouched rows preserved (#1908). _scan = {f for fl in incremental['files'].values() for f in fl} -save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) print('[graphify update] Manifest saved.') -" +" "INPUT_PATH" ``` Then run Steps 4–8 on the merged graph as normal. diff --git a/tools/skillgen/expected/graphify__skills__amp__references__transcribe.md b/tools/skillgen/expected/graphify__skills__amp__references__transcribe.md index b967f83799..bbe8b93e45 100644 --- a/tools/skillgen/expected/graphify__skills__amp__references__transcribe.md +++ b/tools/skillgen/expected/graphify__skills__amp__references__transcribe.md @@ -8,13 +8,11 @@ Skip this step entirely if `detect` returned zero `video` files. Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. -**Strategy:** Read the god nodes from `graphify-out/.graphify_detect.json` (or the analysis file if it exists from a previous run). You are already a language model — write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. - -**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` +**Strategy:** If `graphify-out/.graphify_analysis.json` exists from a previous run, read its top god-node labels and write a one-sentence domain hint from them. Otherwise use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` Pass the chosen prompt to Whisper as its initial prompt. No separate API call needed. **Step 1 - Write the Whisper prompt yourself.** -Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: +Read the top god-node labels from previous analysis when available, then compose a short domain hint sentence. With no previous analysis, use the generic fallback above. - Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` - Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` diff --git a/tools/skillgen/expected/graphify__skills__amp__references__update.md b/tools/skillgen/expected/graphify__skills__amp__references__update.md index bd8e659d3c..058ca8e6d2 100644 --- a/tools/skillgen/expected/graphify__skills__amp__references__update.md +++ b/tools/skillgen/expected/graphify__skills__amp__references__update.md @@ -12,7 +12,7 @@ import sys, json from graphify.detect import detect_incremental, save_manifest from pathlib import Path -result = detect_incremental(Path('INPUT_PATH')) +result = detect_incremental(Path(sys.argv[1])) new_total = result.get('new_total', 0) print(json.dumps(result, indent=2, ensure_ascii=False)) Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") @@ -24,7 +24,7 @@ if deleted: print(f'{len(deleted)} deleted file(s) to prune.') if new_total > 0: print(f'{new_total} new/changed file(s) to re-extract.') -" +" "INPUT_PATH" ``` Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: @@ -84,7 +84,7 @@ Then: ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from pathlib import Path from graphify.build import build_merge from graphify.detect import save_manifest @@ -113,7 +113,7 @@ G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, - root='INPUT_PATH', + root=sys.argv[1], directed=IS_DIRECTED, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -150,7 +150,7 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) # is lost forever (#2015). Mirrors the library extract path # (cli._stamped_manifest_files + clear_semantic + scan_corpus). from graphify.cli import _stamped_manifest_files -_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) +_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path(sys.argv[1])) # Changed semantic files dispatched this run but NOT stamped had their chunk fail # or be omitted; clear any stale semantic_hash so they are re-queued (#1948). _sem_types = ('document', 'paper', 'image') @@ -160,9 +160,9 @@ _cleared = _dispatched - _stamped # scan_corpus = the RAW full corpus so in-root files newly excluded since last run # are dropped rather than masquerading as deletions; untouched rows preserved (#1908). _scan = {f for fl in incremental['files'].values() for f in fl} -save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) print('[graphify update] Manifest saved.') -" +" "INPUT_PATH" ``` Then run Steps 4–8 on the merged graph as normal. diff --git a/tools/skillgen/expected/graphify__skills__claude__references__transcribe.md b/tools/skillgen/expected/graphify__skills__claude__references__transcribe.md index b967f83799..bbe8b93e45 100644 --- a/tools/skillgen/expected/graphify__skills__claude__references__transcribe.md +++ b/tools/skillgen/expected/graphify__skills__claude__references__transcribe.md @@ -8,13 +8,11 @@ Skip this step entirely if `detect` returned zero `video` files. Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. -**Strategy:** Read the god nodes from `graphify-out/.graphify_detect.json` (or the analysis file if it exists from a previous run). You are already a language model — write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. - -**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` +**Strategy:** If `graphify-out/.graphify_analysis.json` exists from a previous run, read its top god-node labels and write a one-sentence domain hint from them. Otherwise use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` Pass the chosen prompt to Whisper as its initial prompt. No separate API call needed. **Step 1 - Write the Whisper prompt yourself.** -Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: +Read the top god-node labels from previous analysis when available, then compose a short domain hint sentence. With no previous analysis, use the generic fallback above. - Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` - Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` diff --git a/tools/skillgen/expected/graphify__skills__claude__references__update.md b/tools/skillgen/expected/graphify__skills__claude__references__update.md index bd8e659d3c..058ca8e6d2 100644 --- a/tools/skillgen/expected/graphify__skills__claude__references__update.md +++ b/tools/skillgen/expected/graphify__skills__claude__references__update.md @@ -12,7 +12,7 @@ import sys, json from graphify.detect import detect_incremental, save_manifest from pathlib import Path -result = detect_incremental(Path('INPUT_PATH')) +result = detect_incremental(Path(sys.argv[1])) new_total = result.get('new_total', 0) print(json.dumps(result, indent=2, ensure_ascii=False)) Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") @@ -24,7 +24,7 @@ if deleted: print(f'{len(deleted)} deleted file(s) to prune.') if new_total > 0: print(f'{new_total} new/changed file(s) to re-extract.') -" +" "INPUT_PATH" ``` Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: @@ -84,7 +84,7 @@ Then: ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from pathlib import Path from graphify.build import build_merge from graphify.detect import save_manifest @@ -113,7 +113,7 @@ G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, - root='INPUT_PATH', + root=sys.argv[1], directed=IS_DIRECTED, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -150,7 +150,7 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) # is lost forever (#2015). Mirrors the library extract path # (cli._stamped_manifest_files + clear_semantic + scan_corpus). from graphify.cli import _stamped_manifest_files -_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) +_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path(sys.argv[1])) # Changed semantic files dispatched this run but NOT stamped had their chunk fail # or be omitted; clear any stale semantic_hash so they are re-queued (#1948). _sem_types = ('document', 'paper', 'image') @@ -160,9 +160,9 @@ _cleared = _dispatched - _stamped # scan_corpus = the RAW full corpus so in-root files newly excluded since last run # are dropped rather than masquerading as deletions; untouched rows preserved (#1908). _scan = {f for fl in incremental['files'].values() for f in fl} -save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) print('[graphify update] Manifest saved.') -" +" "INPUT_PATH" ``` Then run Steps 4–8 on the merged graph as normal. diff --git a/tools/skillgen/expected/graphify__skills__claw__references__transcribe.md b/tools/skillgen/expected/graphify__skills__claw__references__transcribe.md index b967f83799..bbe8b93e45 100644 --- a/tools/skillgen/expected/graphify__skills__claw__references__transcribe.md +++ b/tools/skillgen/expected/graphify__skills__claw__references__transcribe.md @@ -8,13 +8,11 @@ Skip this step entirely if `detect` returned zero `video` files. Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. -**Strategy:** Read the god nodes from `graphify-out/.graphify_detect.json` (or the analysis file if it exists from a previous run). You are already a language model — write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. - -**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` +**Strategy:** If `graphify-out/.graphify_analysis.json` exists from a previous run, read its top god-node labels and write a one-sentence domain hint from them. Otherwise use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` Pass the chosen prompt to Whisper as its initial prompt. No separate API call needed. **Step 1 - Write the Whisper prompt yourself.** -Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: +Read the top god-node labels from previous analysis when available, then compose a short domain hint sentence. With no previous analysis, use the generic fallback above. - Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` - Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` diff --git a/tools/skillgen/expected/graphify__skills__claw__references__update.md b/tools/skillgen/expected/graphify__skills__claw__references__update.md index bd8e659d3c..058ca8e6d2 100644 --- a/tools/skillgen/expected/graphify__skills__claw__references__update.md +++ b/tools/skillgen/expected/graphify__skills__claw__references__update.md @@ -12,7 +12,7 @@ import sys, json from graphify.detect import detect_incremental, save_manifest from pathlib import Path -result = detect_incremental(Path('INPUT_PATH')) +result = detect_incremental(Path(sys.argv[1])) new_total = result.get('new_total', 0) print(json.dumps(result, indent=2, ensure_ascii=False)) Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") @@ -24,7 +24,7 @@ if deleted: print(f'{len(deleted)} deleted file(s) to prune.') if new_total > 0: print(f'{new_total} new/changed file(s) to re-extract.') -" +" "INPUT_PATH" ``` Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: @@ -84,7 +84,7 @@ Then: ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from pathlib import Path from graphify.build import build_merge from graphify.detect import save_manifest @@ -113,7 +113,7 @@ G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, - root='INPUT_PATH', + root=sys.argv[1], directed=IS_DIRECTED, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -150,7 +150,7 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) # is lost forever (#2015). Mirrors the library extract path # (cli._stamped_manifest_files + clear_semantic + scan_corpus). from graphify.cli import _stamped_manifest_files -_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) +_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path(sys.argv[1])) # Changed semantic files dispatched this run but NOT stamped had their chunk fail # or be omitted; clear any stale semantic_hash so they are re-queued (#1948). _sem_types = ('document', 'paper', 'image') @@ -160,9 +160,9 @@ _cleared = _dispatched - _stamped # scan_corpus = the RAW full corpus so in-root files newly excluded since last run # are dropped rather than masquerading as deletions; untouched rows preserved (#1908). _scan = {f for fl in incremental['files'].values() for f in fl} -save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) print('[graphify update] Manifest saved.') -" +" "INPUT_PATH" ``` Then run Steps 4–8 on the merged graph as normal. diff --git a/tools/skillgen/expected/graphify__skills__codex__references__transcribe.md b/tools/skillgen/expected/graphify__skills__codex__references__transcribe.md index b967f83799..bbe8b93e45 100644 --- a/tools/skillgen/expected/graphify__skills__codex__references__transcribe.md +++ b/tools/skillgen/expected/graphify__skills__codex__references__transcribe.md @@ -8,13 +8,11 @@ Skip this step entirely if `detect` returned zero `video` files. Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. -**Strategy:** Read the god nodes from `graphify-out/.graphify_detect.json` (or the analysis file if it exists from a previous run). You are already a language model — write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. - -**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` +**Strategy:** If `graphify-out/.graphify_analysis.json` exists from a previous run, read its top god-node labels and write a one-sentence domain hint from them. Otherwise use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` Pass the chosen prompt to Whisper as its initial prompt. No separate API call needed. **Step 1 - Write the Whisper prompt yourself.** -Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: +Read the top god-node labels from previous analysis when available, then compose a short domain hint sentence. With no previous analysis, use the generic fallback above. - Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` - Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` diff --git a/tools/skillgen/expected/graphify__skills__codex__references__update.md b/tools/skillgen/expected/graphify__skills__codex__references__update.md index bd8e659d3c..058ca8e6d2 100644 --- a/tools/skillgen/expected/graphify__skills__codex__references__update.md +++ b/tools/skillgen/expected/graphify__skills__codex__references__update.md @@ -12,7 +12,7 @@ import sys, json from graphify.detect import detect_incremental, save_manifest from pathlib import Path -result = detect_incremental(Path('INPUT_PATH')) +result = detect_incremental(Path(sys.argv[1])) new_total = result.get('new_total', 0) print(json.dumps(result, indent=2, ensure_ascii=False)) Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") @@ -24,7 +24,7 @@ if deleted: print(f'{len(deleted)} deleted file(s) to prune.') if new_total > 0: print(f'{new_total} new/changed file(s) to re-extract.') -" +" "INPUT_PATH" ``` Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: @@ -84,7 +84,7 @@ Then: ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from pathlib import Path from graphify.build import build_merge from graphify.detect import save_manifest @@ -113,7 +113,7 @@ G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, - root='INPUT_PATH', + root=sys.argv[1], directed=IS_DIRECTED, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -150,7 +150,7 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) # is lost forever (#2015). Mirrors the library extract path # (cli._stamped_manifest_files + clear_semantic + scan_corpus). from graphify.cli import _stamped_manifest_files -_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) +_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path(sys.argv[1])) # Changed semantic files dispatched this run but NOT stamped had their chunk fail # or be omitted; clear any stale semantic_hash so they are re-queued (#1948). _sem_types = ('document', 'paper', 'image') @@ -160,9 +160,9 @@ _cleared = _dispatched - _stamped # scan_corpus = the RAW full corpus so in-root files newly excluded since last run # are dropped rather than masquerading as deletions; untouched rows preserved (#1908). _scan = {f for fl in incremental['files'].values() for f in fl} -save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) print('[graphify update] Manifest saved.') -" +" "INPUT_PATH" ``` Then run Steps 4–8 on the merged graph as normal. diff --git a/tools/skillgen/expected/graphify__skills__copilot__references__transcribe.md b/tools/skillgen/expected/graphify__skills__copilot__references__transcribe.md index b967f83799..bbe8b93e45 100644 --- a/tools/skillgen/expected/graphify__skills__copilot__references__transcribe.md +++ b/tools/skillgen/expected/graphify__skills__copilot__references__transcribe.md @@ -8,13 +8,11 @@ Skip this step entirely if `detect` returned zero `video` files. Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. -**Strategy:** Read the god nodes from `graphify-out/.graphify_detect.json` (or the analysis file if it exists from a previous run). You are already a language model — write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. - -**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` +**Strategy:** If `graphify-out/.graphify_analysis.json` exists from a previous run, read its top god-node labels and write a one-sentence domain hint from them. Otherwise use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` Pass the chosen prompt to Whisper as its initial prompt. No separate API call needed. **Step 1 - Write the Whisper prompt yourself.** -Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: +Read the top god-node labels from previous analysis when available, then compose a short domain hint sentence. With no previous analysis, use the generic fallback above. - Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` - Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` diff --git a/tools/skillgen/expected/graphify__skills__copilot__references__update.md b/tools/skillgen/expected/graphify__skills__copilot__references__update.md index bd8e659d3c..058ca8e6d2 100644 --- a/tools/skillgen/expected/graphify__skills__copilot__references__update.md +++ b/tools/skillgen/expected/graphify__skills__copilot__references__update.md @@ -12,7 +12,7 @@ import sys, json from graphify.detect import detect_incremental, save_manifest from pathlib import Path -result = detect_incremental(Path('INPUT_PATH')) +result = detect_incremental(Path(sys.argv[1])) new_total = result.get('new_total', 0) print(json.dumps(result, indent=2, ensure_ascii=False)) Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") @@ -24,7 +24,7 @@ if deleted: print(f'{len(deleted)} deleted file(s) to prune.') if new_total > 0: print(f'{new_total} new/changed file(s) to re-extract.') -" +" "INPUT_PATH" ``` Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: @@ -84,7 +84,7 @@ Then: ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from pathlib import Path from graphify.build import build_merge from graphify.detect import save_manifest @@ -113,7 +113,7 @@ G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, - root='INPUT_PATH', + root=sys.argv[1], directed=IS_DIRECTED, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -150,7 +150,7 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) # is lost forever (#2015). Mirrors the library extract path # (cli._stamped_manifest_files + clear_semantic + scan_corpus). from graphify.cli import _stamped_manifest_files -_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) +_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path(sys.argv[1])) # Changed semantic files dispatched this run but NOT stamped had their chunk fail # or be omitted; clear any stale semantic_hash so they are re-queued (#1948). _sem_types = ('document', 'paper', 'image') @@ -160,9 +160,9 @@ _cleared = _dispatched - _stamped # scan_corpus = the RAW full corpus so in-root files newly excluded since last run # are dropped rather than masquerading as deletions; untouched rows preserved (#1908). _scan = {f for fl in incremental['files'].values() for f in fl} -save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) print('[graphify update] Manifest saved.') -" +" "INPUT_PATH" ``` Then run Steps 4–8 on the merged graph as normal. diff --git a/tools/skillgen/expected/graphify__skills__droid__references__transcribe.md b/tools/skillgen/expected/graphify__skills__droid__references__transcribe.md index b967f83799..bbe8b93e45 100644 --- a/tools/skillgen/expected/graphify__skills__droid__references__transcribe.md +++ b/tools/skillgen/expected/graphify__skills__droid__references__transcribe.md @@ -8,13 +8,11 @@ Skip this step entirely if `detect` returned zero `video` files. Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. -**Strategy:** Read the god nodes from `graphify-out/.graphify_detect.json` (or the analysis file if it exists from a previous run). You are already a language model — write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. - -**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` +**Strategy:** If `graphify-out/.graphify_analysis.json` exists from a previous run, read its top god-node labels and write a one-sentence domain hint from them. Otherwise use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` Pass the chosen prompt to Whisper as its initial prompt. No separate API call needed. **Step 1 - Write the Whisper prompt yourself.** -Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: +Read the top god-node labels from previous analysis when available, then compose a short domain hint sentence. With no previous analysis, use the generic fallback above. - Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` - Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` diff --git a/tools/skillgen/expected/graphify__skills__droid__references__update.md b/tools/skillgen/expected/graphify__skills__droid__references__update.md index bd8e659d3c..058ca8e6d2 100644 --- a/tools/skillgen/expected/graphify__skills__droid__references__update.md +++ b/tools/skillgen/expected/graphify__skills__droid__references__update.md @@ -12,7 +12,7 @@ import sys, json from graphify.detect import detect_incremental, save_manifest from pathlib import Path -result = detect_incremental(Path('INPUT_PATH')) +result = detect_incremental(Path(sys.argv[1])) new_total = result.get('new_total', 0) print(json.dumps(result, indent=2, ensure_ascii=False)) Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") @@ -24,7 +24,7 @@ if deleted: print(f'{len(deleted)} deleted file(s) to prune.') if new_total > 0: print(f'{new_total} new/changed file(s) to re-extract.') -" +" "INPUT_PATH" ``` Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: @@ -84,7 +84,7 @@ Then: ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from pathlib import Path from graphify.build import build_merge from graphify.detect import save_manifest @@ -113,7 +113,7 @@ G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, - root='INPUT_PATH', + root=sys.argv[1], directed=IS_DIRECTED, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -150,7 +150,7 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) # is lost forever (#2015). Mirrors the library extract path # (cli._stamped_manifest_files + clear_semantic + scan_corpus). from graphify.cli import _stamped_manifest_files -_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) +_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path(sys.argv[1])) # Changed semantic files dispatched this run but NOT stamped had their chunk fail # or be omitted; clear any stale semantic_hash so they are re-queued (#1948). _sem_types = ('document', 'paper', 'image') @@ -160,9 +160,9 @@ _cleared = _dispatched - _stamped # scan_corpus = the RAW full corpus so in-root files newly excluded since last run # are dropped rather than masquerading as deletions; untouched rows preserved (#1908). _scan = {f for fl in incremental['files'].values() for f in fl} -save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) print('[graphify update] Manifest saved.') -" +" "INPUT_PATH" ``` Then run Steps 4–8 on the merged graph as normal. diff --git a/tools/skillgen/expected/graphify__skills__kilo__references__transcribe.md b/tools/skillgen/expected/graphify__skills__kilo__references__transcribe.md index b967f83799..bbe8b93e45 100644 --- a/tools/skillgen/expected/graphify__skills__kilo__references__transcribe.md +++ b/tools/skillgen/expected/graphify__skills__kilo__references__transcribe.md @@ -8,13 +8,11 @@ Skip this step entirely if `detect` returned zero `video` files. Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. -**Strategy:** Read the god nodes from `graphify-out/.graphify_detect.json` (or the analysis file if it exists from a previous run). You are already a language model — write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. - -**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` +**Strategy:** If `graphify-out/.graphify_analysis.json` exists from a previous run, read its top god-node labels and write a one-sentence domain hint from them. Otherwise use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` Pass the chosen prompt to Whisper as its initial prompt. No separate API call needed. **Step 1 - Write the Whisper prompt yourself.** -Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: +Read the top god-node labels from previous analysis when available, then compose a short domain hint sentence. With no previous analysis, use the generic fallback above. - Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` - Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` diff --git a/tools/skillgen/expected/graphify__skills__kilo__references__update.md b/tools/skillgen/expected/graphify__skills__kilo__references__update.md index bd8e659d3c..058ca8e6d2 100644 --- a/tools/skillgen/expected/graphify__skills__kilo__references__update.md +++ b/tools/skillgen/expected/graphify__skills__kilo__references__update.md @@ -12,7 +12,7 @@ import sys, json from graphify.detect import detect_incremental, save_manifest from pathlib import Path -result = detect_incremental(Path('INPUT_PATH')) +result = detect_incremental(Path(sys.argv[1])) new_total = result.get('new_total', 0) print(json.dumps(result, indent=2, ensure_ascii=False)) Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") @@ -24,7 +24,7 @@ if deleted: print(f'{len(deleted)} deleted file(s) to prune.') if new_total > 0: print(f'{new_total} new/changed file(s) to re-extract.') -" +" "INPUT_PATH" ``` Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: @@ -84,7 +84,7 @@ Then: ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from pathlib import Path from graphify.build import build_merge from graphify.detect import save_manifest @@ -113,7 +113,7 @@ G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, - root='INPUT_PATH', + root=sys.argv[1], directed=IS_DIRECTED, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -150,7 +150,7 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) # is lost forever (#2015). Mirrors the library extract path # (cli._stamped_manifest_files + clear_semantic + scan_corpus). from graphify.cli import _stamped_manifest_files -_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) +_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path(sys.argv[1])) # Changed semantic files dispatched this run but NOT stamped had their chunk fail # or be omitted; clear any stale semantic_hash so they are re-queued (#1948). _sem_types = ('document', 'paper', 'image') @@ -160,9 +160,9 @@ _cleared = _dispatched - _stamped # scan_corpus = the RAW full corpus so in-root files newly excluded since last run # are dropped rather than masquerading as deletions; untouched rows preserved (#1908). _scan = {f for fl in incremental['files'].values() for f in fl} -save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) print('[graphify update] Manifest saved.') -" +" "INPUT_PATH" ``` Then run Steps 4–8 on the merged graph as normal. diff --git a/tools/skillgen/expected/graphify__skills__kiro__references__transcribe.md b/tools/skillgen/expected/graphify__skills__kiro__references__transcribe.md index b967f83799..bbe8b93e45 100644 --- a/tools/skillgen/expected/graphify__skills__kiro__references__transcribe.md +++ b/tools/skillgen/expected/graphify__skills__kiro__references__transcribe.md @@ -8,13 +8,11 @@ Skip this step entirely if `detect` returned zero `video` files. Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. -**Strategy:** Read the god nodes from `graphify-out/.graphify_detect.json` (or the analysis file if it exists from a previous run). You are already a language model — write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. - -**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` +**Strategy:** If `graphify-out/.graphify_analysis.json` exists from a previous run, read its top god-node labels and write a one-sentence domain hint from them. Otherwise use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` Pass the chosen prompt to Whisper as its initial prompt. No separate API call needed. **Step 1 - Write the Whisper prompt yourself.** -Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: +Read the top god-node labels from previous analysis when available, then compose a short domain hint sentence. With no previous analysis, use the generic fallback above. - Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` - Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` diff --git a/tools/skillgen/expected/graphify__skills__kiro__references__update.md b/tools/skillgen/expected/graphify__skills__kiro__references__update.md index bd8e659d3c..058ca8e6d2 100644 --- a/tools/skillgen/expected/graphify__skills__kiro__references__update.md +++ b/tools/skillgen/expected/graphify__skills__kiro__references__update.md @@ -12,7 +12,7 @@ import sys, json from graphify.detect import detect_incremental, save_manifest from pathlib import Path -result = detect_incremental(Path('INPUT_PATH')) +result = detect_incremental(Path(sys.argv[1])) new_total = result.get('new_total', 0) print(json.dumps(result, indent=2, ensure_ascii=False)) Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") @@ -24,7 +24,7 @@ if deleted: print(f'{len(deleted)} deleted file(s) to prune.') if new_total > 0: print(f'{new_total} new/changed file(s) to re-extract.') -" +" "INPUT_PATH" ``` Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: @@ -84,7 +84,7 @@ Then: ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from pathlib import Path from graphify.build import build_merge from graphify.detect import save_manifest @@ -113,7 +113,7 @@ G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, - root='INPUT_PATH', + root=sys.argv[1], directed=IS_DIRECTED, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -150,7 +150,7 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) # is lost forever (#2015). Mirrors the library extract path # (cli._stamped_manifest_files + clear_semantic + scan_corpus). from graphify.cli import _stamped_manifest_files -_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) +_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path(sys.argv[1])) # Changed semantic files dispatched this run but NOT stamped had their chunk fail # or be omitted; clear any stale semantic_hash so they are re-queued (#1948). _sem_types = ('document', 'paper', 'image') @@ -160,9 +160,9 @@ _cleared = _dispatched - _stamped # scan_corpus = the RAW full corpus so in-root files newly excluded since last run # are dropped rather than masquerading as deletions; untouched rows preserved (#1908). _scan = {f for fl in incremental['files'].values() for f in fl} -save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) print('[graphify update] Manifest saved.') -" +" "INPUT_PATH" ``` Then run Steps 4–8 on the merged graph as normal. diff --git a/tools/skillgen/expected/graphify__skills__opencode__references__transcribe.md b/tools/skillgen/expected/graphify__skills__opencode__references__transcribe.md index b967f83799..bbe8b93e45 100644 --- a/tools/skillgen/expected/graphify__skills__opencode__references__transcribe.md +++ b/tools/skillgen/expected/graphify__skills__opencode__references__transcribe.md @@ -8,13 +8,11 @@ Skip this step entirely if `detect` returned zero `video` files. Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. -**Strategy:** Read the god nodes from `graphify-out/.graphify_detect.json` (or the analysis file if it exists from a previous run). You are already a language model — write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. - -**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` +**Strategy:** If `graphify-out/.graphify_analysis.json` exists from a previous run, read its top god-node labels and write a one-sentence domain hint from them. Otherwise use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` Pass the chosen prompt to Whisper as its initial prompt. No separate API call needed. **Step 1 - Write the Whisper prompt yourself.** -Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: +Read the top god-node labels from previous analysis when available, then compose a short domain hint sentence. With no previous analysis, use the generic fallback above. - Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` - Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` diff --git a/tools/skillgen/expected/graphify__skills__opencode__references__update.md b/tools/skillgen/expected/graphify__skills__opencode__references__update.md index bd8e659d3c..058ca8e6d2 100644 --- a/tools/skillgen/expected/graphify__skills__opencode__references__update.md +++ b/tools/skillgen/expected/graphify__skills__opencode__references__update.md @@ -12,7 +12,7 @@ import sys, json from graphify.detect import detect_incremental, save_manifest from pathlib import Path -result = detect_incremental(Path('INPUT_PATH')) +result = detect_incremental(Path(sys.argv[1])) new_total = result.get('new_total', 0) print(json.dumps(result, indent=2, ensure_ascii=False)) Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") @@ -24,7 +24,7 @@ if deleted: print(f'{len(deleted)} deleted file(s) to prune.') if new_total > 0: print(f'{new_total} new/changed file(s) to re-extract.') -" +" "INPUT_PATH" ``` Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: @@ -84,7 +84,7 @@ Then: ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from pathlib import Path from graphify.build import build_merge from graphify.detect import save_manifest @@ -113,7 +113,7 @@ G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, - root='INPUT_PATH', + root=sys.argv[1], directed=IS_DIRECTED, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -150,7 +150,7 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) # is lost forever (#2015). Mirrors the library extract path # (cli._stamped_manifest_files + clear_semantic + scan_corpus). from graphify.cli import _stamped_manifest_files -_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) +_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path(sys.argv[1])) # Changed semantic files dispatched this run but NOT stamped had their chunk fail # or be omitted; clear any stale semantic_hash so they are re-queued (#1948). _sem_types = ('document', 'paper', 'image') @@ -160,9 +160,9 @@ _cleared = _dispatched - _stamped # scan_corpus = the RAW full corpus so in-root files newly excluded since last run # are dropped rather than masquerading as deletions; untouched rows preserved (#1908). _scan = {f for fl in incremental['files'].values() for f in fl} -save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) print('[graphify update] Manifest saved.') -" +" "INPUT_PATH" ``` Then run Steps 4–8 on the merged graph as normal. diff --git a/tools/skillgen/expected/graphify__skills__pi__references__transcribe.md b/tools/skillgen/expected/graphify__skills__pi__references__transcribe.md index b967f83799..bbe8b93e45 100644 --- a/tools/skillgen/expected/graphify__skills__pi__references__transcribe.md +++ b/tools/skillgen/expected/graphify__skills__pi__references__transcribe.md @@ -8,13 +8,11 @@ Skip this step entirely if `detect` returned zero `video` files. Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. -**Strategy:** Read the god nodes from `graphify-out/.graphify_detect.json` (or the analysis file if it exists from a previous run). You are already a language model — write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. - -**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` +**Strategy:** If `graphify-out/.graphify_analysis.json` exists from a previous run, read its top god-node labels and write a one-sentence domain hint from them. Otherwise use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` Pass the chosen prompt to Whisper as its initial prompt. No separate API call needed. **Step 1 - Write the Whisper prompt yourself.** -Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: +Read the top god-node labels from previous analysis when available, then compose a short domain hint sentence. With no previous analysis, use the generic fallback above. - Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` - Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` diff --git a/tools/skillgen/expected/graphify__skills__pi__references__update.md b/tools/skillgen/expected/graphify__skills__pi__references__update.md index bd8e659d3c..058ca8e6d2 100644 --- a/tools/skillgen/expected/graphify__skills__pi__references__update.md +++ b/tools/skillgen/expected/graphify__skills__pi__references__update.md @@ -12,7 +12,7 @@ import sys, json from graphify.detect import detect_incremental, save_manifest from pathlib import Path -result = detect_incremental(Path('INPUT_PATH')) +result = detect_incremental(Path(sys.argv[1])) new_total = result.get('new_total', 0) print(json.dumps(result, indent=2, ensure_ascii=False)) Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") @@ -24,7 +24,7 @@ if deleted: print(f'{len(deleted)} deleted file(s) to prune.') if new_total > 0: print(f'{new_total} new/changed file(s) to re-extract.') -" +" "INPUT_PATH" ``` Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: @@ -84,7 +84,7 @@ Then: ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from pathlib import Path from graphify.build import build_merge from graphify.detect import save_manifest @@ -113,7 +113,7 @@ G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, - root='INPUT_PATH', + root=sys.argv[1], directed=IS_DIRECTED, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -150,7 +150,7 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) # is lost forever (#2015). Mirrors the library extract path # (cli._stamped_manifest_files + clear_semantic + scan_corpus). from graphify.cli import _stamped_manifest_files -_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) +_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path(sys.argv[1])) # Changed semantic files dispatched this run but NOT stamped had their chunk fail # or be omitted; clear any stale semantic_hash so they are re-queued (#1948). _sem_types = ('document', 'paper', 'image') @@ -160,9 +160,9 @@ _cleared = _dispatched - _stamped # scan_corpus = the RAW full corpus so in-root files newly excluded since last run # are dropped rather than masquerading as deletions; untouched rows preserved (#1908). _scan = {f for fl in incremental['files'].values() for f in fl} -save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) print('[graphify update] Manifest saved.') -" +" "INPUT_PATH" ``` Then run Steps 4–8 on the merged graph as normal. diff --git a/tools/skillgen/expected/graphify__skills__trae__references__transcribe.md b/tools/skillgen/expected/graphify__skills__trae__references__transcribe.md index b967f83799..bbe8b93e45 100644 --- a/tools/skillgen/expected/graphify__skills__trae__references__transcribe.md +++ b/tools/skillgen/expected/graphify__skills__trae__references__transcribe.md @@ -8,13 +8,11 @@ Skip this step entirely if `detect` returned zero `video` files. Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. -**Strategy:** Read the god nodes from `graphify-out/.graphify_detect.json` (or the analysis file if it exists from a previous run). You are already a language model — write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. - -**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` +**Strategy:** If `graphify-out/.graphify_analysis.json` exists from a previous run, read its top god-node labels and write a one-sentence domain hint from them. Otherwise use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` Pass the chosen prompt to Whisper as its initial prompt. No separate API call needed. **Step 1 - Write the Whisper prompt yourself.** -Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: +Read the top god-node labels from previous analysis when available, then compose a short domain hint sentence. With no previous analysis, use the generic fallback above. - Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` - Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` diff --git a/tools/skillgen/expected/graphify__skills__trae__references__update.md b/tools/skillgen/expected/graphify__skills__trae__references__update.md index bd8e659d3c..058ca8e6d2 100644 --- a/tools/skillgen/expected/graphify__skills__trae__references__update.md +++ b/tools/skillgen/expected/graphify__skills__trae__references__update.md @@ -12,7 +12,7 @@ import sys, json from graphify.detect import detect_incremental, save_manifest from pathlib import Path -result = detect_incremental(Path('INPUT_PATH')) +result = detect_incremental(Path(sys.argv[1])) new_total = result.get('new_total', 0) print(json.dumps(result, indent=2, ensure_ascii=False)) Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") @@ -24,7 +24,7 @@ if deleted: print(f'{len(deleted)} deleted file(s) to prune.') if new_total > 0: print(f'{new_total} new/changed file(s) to re-extract.') -" +" "INPUT_PATH" ``` Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: @@ -84,7 +84,7 @@ Then: ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from pathlib import Path from graphify.build import build_merge from graphify.detect import save_manifest @@ -113,7 +113,7 @@ G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, - root='INPUT_PATH', + root=sys.argv[1], directed=IS_DIRECTED, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -150,7 +150,7 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) # is lost forever (#2015). Mirrors the library extract path # (cli._stamped_manifest_files + clear_semantic + scan_corpus). from graphify.cli import _stamped_manifest_files -_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) +_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path(sys.argv[1])) # Changed semantic files dispatched this run but NOT stamped had their chunk fail # or be omitted; clear any stale semantic_hash so they are re-queued (#1948). _sem_types = ('document', 'paper', 'image') @@ -160,9 +160,9 @@ _cleared = _dispatched - _stamped # scan_corpus = the RAW full corpus so in-root files newly excluded since last run # are dropped rather than masquerading as deletions; untouched rows preserved (#1908). _scan = {f for fl in incremental['files'].values() for f in fl} -save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) print('[graphify update] Manifest saved.') -" +" "INPUT_PATH" ``` Then run Steps 4–8 on the merged graph as normal. diff --git a/tools/skillgen/expected/graphify__skills__vscode__references__transcribe.md b/tools/skillgen/expected/graphify__skills__vscode__references__transcribe.md index b967f83799..bbe8b93e45 100644 --- a/tools/skillgen/expected/graphify__skills__vscode__references__transcribe.md +++ b/tools/skillgen/expected/graphify__skills__vscode__references__transcribe.md @@ -8,13 +8,11 @@ Skip this step entirely if `detect` returned zero `video` files. Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. -**Strategy:** Read the god nodes from `graphify-out/.graphify_detect.json` (or the analysis file if it exists from a previous run). You are already a language model — write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. - -**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` +**Strategy:** If `graphify-out/.graphify_analysis.json` exists from a previous run, read its top god-node labels and write a one-sentence domain hint from them. Otherwise use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` Pass the chosen prompt to Whisper as its initial prompt. No separate API call needed. **Step 1 - Write the Whisper prompt yourself.** -Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: +Read the top god-node labels from previous analysis when available, then compose a short domain hint sentence. With no previous analysis, use the generic fallback above. - Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` - Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` diff --git a/tools/skillgen/expected/graphify__skills__vscode__references__update.md b/tools/skillgen/expected/graphify__skills__vscode__references__update.md index bd8e659d3c..058ca8e6d2 100644 --- a/tools/skillgen/expected/graphify__skills__vscode__references__update.md +++ b/tools/skillgen/expected/graphify__skills__vscode__references__update.md @@ -12,7 +12,7 @@ import sys, json from graphify.detect import detect_incremental, save_manifest from pathlib import Path -result = detect_incremental(Path('INPUT_PATH')) +result = detect_incremental(Path(sys.argv[1])) new_total = result.get('new_total', 0) print(json.dumps(result, indent=2, ensure_ascii=False)) Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") @@ -24,7 +24,7 @@ if deleted: print(f'{len(deleted)} deleted file(s) to prune.') if new_total > 0: print(f'{new_total} new/changed file(s) to re-extract.') -" +" "INPUT_PATH" ``` Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: @@ -84,7 +84,7 @@ Then: ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from pathlib import Path from graphify.build import build_merge from graphify.detect import save_manifest @@ -113,7 +113,7 @@ G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, - root='INPUT_PATH', + root=sys.argv[1], directed=IS_DIRECTED, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -150,7 +150,7 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) # is lost forever (#2015). Mirrors the library extract path # (cli._stamped_manifest_files + clear_semantic + scan_corpus). from graphify.cli import _stamped_manifest_files -_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) +_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path(sys.argv[1])) # Changed semantic files dispatched this run but NOT stamped had their chunk fail # or be omitted; clear any stale semantic_hash so they are re-queued (#1948). _sem_types = ('document', 'paper', 'image') @@ -160,9 +160,9 @@ _cleared = _dispatched - _stamped # scan_corpus = the RAW full corpus so in-root files newly excluded since last run # are dropped rather than masquerading as deletions; untouched rows preserved (#1908). _scan = {f for fl in incremental['files'].values() for f in fl} -save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) print('[graphify update] Manifest saved.') -" +" "INPUT_PATH" ``` Then run Steps 4–8 on the merged graph as normal. diff --git a/tools/skillgen/expected/graphify__skills__windows__references__transcribe.md b/tools/skillgen/expected/graphify__skills__windows__references__transcribe.md index b967f83799..bbe8b93e45 100644 --- a/tools/skillgen/expected/graphify__skills__windows__references__transcribe.md +++ b/tools/skillgen/expected/graphify__skills__windows__references__transcribe.md @@ -8,13 +8,11 @@ Skip this step entirely if `detect` returned zero `video` files. Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. -**Strategy:** Read the god nodes from `graphify-out/.graphify_detect.json` (or the analysis file if it exists from a previous run). You are already a language model — write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. - -**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` +**Strategy:** If `graphify-out/.graphify_analysis.json` exists from a previous run, read its top god-node labels and write a one-sentence domain hint from them. Otherwise use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` Pass the chosen prompt to Whisper as its initial prompt. No separate API call needed. **Step 1 - Write the Whisper prompt yourself.** -Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: +Read the top god-node labels from previous analysis when available, then compose a short domain hint sentence. With no previous analysis, use the generic fallback above. - Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` - Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` diff --git a/tools/skillgen/expected/graphify__skills__windows__references__update.md b/tools/skillgen/expected/graphify__skills__windows__references__update.md index bd8e659d3c..058ca8e6d2 100644 --- a/tools/skillgen/expected/graphify__skills__windows__references__update.md +++ b/tools/skillgen/expected/graphify__skills__windows__references__update.md @@ -12,7 +12,7 @@ import sys, json from graphify.detect import detect_incremental, save_manifest from pathlib import Path -result = detect_incremental(Path('INPUT_PATH')) +result = detect_incremental(Path(sys.argv[1])) new_total = result.get('new_total', 0) print(json.dumps(result, indent=2, ensure_ascii=False)) Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") @@ -24,7 +24,7 @@ if deleted: print(f'{len(deleted)} deleted file(s) to prune.') if new_total > 0: print(f'{new_total} new/changed file(s) to re-extract.') -" +" "INPUT_PATH" ``` Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: @@ -84,7 +84,7 @@ Then: ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from pathlib import Path from graphify.build import build_merge from graphify.detect import save_manifest @@ -113,7 +113,7 @@ G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, - root='INPUT_PATH', + root=sys.argv[1], directed=IS_DIRECTED, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -150,7 +150,7 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) # is lost forever (#2015). Mirrors the library extract path # (cli._stamped_manifest_files + clear_semantic + scan_corpus). from graphify.cli import _stamped_manifest_files -_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) +_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path(sys.argv[1])) # Changed semantic files dispatched this run but NOT stamped had their chunk fail # or be omitted; clear any stale semantic_hash so they are re-queued (#1948). _sem_types = ('document', 'paper', 'image') @@ -160,9 +160,9 @@ _cleared = _dispatched - _stamped # scan_corpus = the RAW full corpus so in-root files newly excluded since last run # are dropped rather than masquerading as deletions; untouched rows preserved (#1908). _scan = {f for fl in incremental['files'].values() for f in fl} -save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) print('[graphify update] Manifest saved.') -" +" "INPUT_PATH" ``` Then run Steps 4–8 on the merged graph as normal. diff --git a/tools/skillgen/fragments/core/aider.md b/tools/skillgen/fragments/core/aider.md index 4c5f74c88b..085be97cd2 100644 --- a/tools/skillgen/fragments/core/aider.md +++ b/tools/skillgen/fragments/core/aider.md @@ -101,11 +101,12 @@ If the import succeeds, print nothing and move straight to Step 2. ```bash $(cat graphify-out/.graphify_python) -c " import json +import sys from graphify.detect import detect from pathlib import Path -result = detect(Path('INPUT_PATH')) +result = detect(Path(sys.argv[1])) print(json.dumps(result)) -" > .graphify_detect.json +" "INPUT_PATH" > .graphify_detect.json ``` Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: @@ -133,13 +134,11 @@ Skip this step entirely if `detect` returned zero `video` files. Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. -**Strategy:** Read the god nodes from the detect output or analysis file. You are already a language model - write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. - -**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` +**Strategy:** If an analysis file exists from a previous run, read its top god-node labels and write a one-sentence domain hint from them. Otherwise use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` Pass the chosen prompt to Whisper as its initial prompt. No separate API call needed. **Step 1 - Write the Whisper prompt yourself.** -Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: +Read the top god-node labels from previous analysis when available, then compose a short domain hint sentence. With no previous analysis, use the generic fallback above. - Labels: `transformer, attention, encoder, decoder` -> `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` - Labels: `kubernetes, deployment, pod, helm` -> `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` @@ -426,7 +425,7 @@ wrote = to_json(G, communities, 'graphify-out/graph.json') if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (fewer nodes than the existing graph). Run a full rebuild to be safe.') raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, sys.argv[1], suggested_questions=questions) Path('graphify-out/GRAPH_REPORT.md').write_text(report) analysis = { @@ -438,7 +437,7 @@ analysis = { } Path('.graphify_analysis.json').write_text(json.dumps(analysis, indent=2)) print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" +" "INPUT_PATH" ``` If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. @@ -476,7 +475,7 @@ labels = LABELS_DICT # Regenerate questions with real community labels (labels affect question phrasing) questions = suggest_questions(G, communities, labels) -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, sys.argv[1], suggested_questions=questions) # Re-export so graph.json nodes carry the curated community_name (#2490). # Same extraction as Step 4, so the #479 shrink-guard passes on node count; # if it still refuses, surface the guard message - do not force past it. @@ -487,7 +486,7 @@ if not wrote: Path('graphify-out/GRAPH_REPORT.md').write_text(report) Path('.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()})) print('Report updated with community labels') -" +" "INPUT_PATH" ``` Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). @@ -679,6 +678,7 @@ Print the output directly in chat. If `total_words <= 5000`, skip silently - the ```bash $(cat graphify-out/.graphify_python) -c " import json +import sys from pathlib import Path from datetime import datetime, timezone from graphify.detect import save_manifest @@ -689,13 +689,13 @@ extract = json.loads(Path('.graphify_extract.json').read_text()) # Stamp only semantic files that produced output so a failed chunk is re-queued next run, not lost (#2015). from graphify.cli import _stamped_manifest_files _corpus = detect.get('all_files') or detect['files'] -_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) +_manifest_files = _stamped_manifest_files(_corpus, extract, Path(sys.argv[1])) _sem_types = ('document', 'paper', 'image') _dispatched = {f for t, fl in detect['files'].items() if t in _sem_types for f in fl} _stamped = {f for fl in _manifest_files.values() for f in fl} _cleared = _dispatched - _stamped _scan = {f for fl in _corpus.values() for f in fl} -save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) # Update cumulative cost tracker input_tok = extract.get('input_tokens', 0) @@ -719,7 +719,7 @@ cost_path.write_text(json.dumps(cost, indent=2)) print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" +" "INPUT_PATH" rm -f .graphify_detect.json .graphify_extract.json .graphify_ast.json .graphify_semantic.json .graphify_analysis.json .graphify_labels.json; find . -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null rm -f graphify-out/needs_update 2>/dev/null || true ``` @@ -765,7 +765,7 @@ import sys, json from graphify.detect import detect_incremental, save_manifest from pathlib import Path -result = detect_incremental(Path('INPUT_PATH')) +result = detect_incremental(Path(sys.argv[1])) new_total = result.get('new_total', 0) print(json.dumps(result, indent=2)) Path('.graphify_incremental.json').write_text(json.dumps(result)) @@ -777,7 +777,7 @@ if deleted: print(f'{len(deleted)} deleted file(s) to prune.') if new_total > 0: print(f'{new_total} new/changed file(s) to re-extract.') -" +" "INPUT_PATH" ``` If new files exist, first check whether all changed files are code files: diff --git a/tools/skillgen/fragments/core/core.md b/tools/skillgen/fragments/core/core.md index 437f156447..f295ab8c6d 100644 --- a/tools/skillgen/fragments/core/core.md +++ b/tools/skillgen/fragments/core/core.md @@ -67,15 +67,15 @@ Only when the path is one or more `https://github.com/...` URLs, or several loca ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from graphify.detect import detect from pathlib import Path -result = detect(Path('INPUT_PATH')) +result = detect(Path(sys.argv[1])) # Write the sidecar from Python, not a shell redirect, so the same block renders # on PowerShell hosts without console-encoding drift (#2528). Path('graphify-out/.graphify_detect.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") print(f'Detected {result[\"total_files\"]} files') -" +" "INPUT_PATH" ``` Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: @@ -142,13 +142,13 @@ for f in detect.get('files', {}).get('code', []): code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) if code_files: - result = extract(code_files, cache_root=Path('INPUT_PATH')) + result = extract(code_files, cache_root=Path(sys.argv[1])) Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') else: Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") print('No code files - skipping AST extraction') -" +" "INPUT_PATH" ``` #### Part B - Semantic extraction (parallel subagents) @@ -163,7 +163,7 @@ Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],' " ``` -**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** +**MANDATORY: You MUST use the subagent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the subagent tool you are doing this wrong.** Before dispatching subagents, print a timing estimate: - Load `total_words` and file counts from `graphify-out/.graphify_detect.json` @@ -179,7 +179,7 @@ SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from graphify.cache import check_semantic_cache from pathlib import Path @@ -189,7 +189,7 @@ detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encodin # every source file (#1392). Video is transcribed to a document in Step 2.5 first. all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root=sys.argv[1], prompt_file=sys.argv[2]) # Always (re)write the cache file: write hits, else DELETE any leftover from a prior # run so Part C never merges a stale .graphify_cached.json (#1392). @@ -199,7 +199,7 @@ else: Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" +" "INPUT_PATH" "SPEC_PATH" ``` Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. @@ -220,7 +220,7 @@ Wait for all subagents. For each result: If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: +Merge all chunk files into `.graphify_semantic_new.json`. **After each subagent call completes, read the real token counts from the subagent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: ```bash $(cat graphify-out/.graphify_python) -c " import json, glob @@ -247,15 +247,15 @@ print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens' Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root=sys.argv[1], allowed_source_files=uncached, prompt_file=sys.argv[2]) print(f'Cached {saved} files') -" +" "INPUT_PATH" "SPEC_PATH" ``` Merge cached + new results into `graphify-out/.graphify_semantic.json`: @@ -344,7 +344,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -369,7 +369,7 @@ if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') print('If this shrink is intentional (you deleted files), re-run a full build with --force.') raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, sys.argv[1], suggested_questions=questions) Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") analysis = { 'communities': {str(k): v for k, v in communities.items()}, @@ -380,7 +380,7 @@ analysis = { } Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" +" "INPUT_PATH" ``` If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. @@ -393,12 +393,12 @@ A non-destructive diagnostic on the extraction, before labeling. It surfaces edg ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from pathlib import Path from graphify.diagnostics import diagnose_extraction, format_diagnostic_report extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') +summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root=sys.argv[1]) print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( ('dangling_endpoint_edges', 'dangling-endpoint edges'), @@ -408,7 +408,7 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0)] print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') -" +" "INPUT_PATH" ``` Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules). @@ -434,7 +434,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} @@ -445,7 +445,7 @@ labels = LABELS_DICT # Regenerate questions with real community labels (labels affect question phrasing) questions = suggest_questions(G, communities, labels) -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, sys.argv[1], suggested_questions=questions) # Re-export so graph.json nodes carry the curated community_name (#2490). # Same extraction as Step 4, so the #479 shrink-guard passes on node count; # if it still refuses, surface the guard message - do not force past it. @@ -457,7 +457,7 @@ if not wrote: Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") print('Report updated with community labels') -" +" "INPUT_PATH" ``` Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). @@ -492,7 +492,7 @@ These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, ` ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from pathlib import Path from datetime import datetime, timezone from graphify.detect import save_manifest @@ -515,7 +515,7 @@ extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encod # types are gated on output. from graphify.cli import _stamped_manifest_files _corpus = detect.get('all_files') or detect['files'] -_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) +_manifest_files = _stamped_manifest_files(_corpus, extract, Path(sys.argv[1])) # Files dispatched this run (the changed subset) but NOT stamped above still carry # a stale semantic_hash from a prior run; clear it so detect_incremental re-queues # them instead of reading them as unchanged (#1948). @@ -527,7 +527,7 @@ _cleared = _dispatched - _stamped # files newly excluded since last run are dropped rather than masquerading as # deletions; untouched files' prior rows are still preserved (#1908). _scan = {f for fl in _corpus.values() for f in fl} -save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) # Update cumulative cost tracker input_tok = extract.get('input_tokens', 0) @@ -551,7 +551,7 @@ cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\" print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" +" "INPUT_PATH" rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null rm -f graphify-out/needs_update 2>/dev/null || true diff --git a/tools/skillgen/fragments/core/devin.md b/tools/skillgen/fragments/core/devin.md index cfb5aa7b13..bea72f7eea 100644 --- a/tools/skillgen/fragments/core/devin.md +++ b/tools/skillgen/fragments/core/devin.md @@ -114,11 +114,12 @@ If the import succeeds, print nothing and move straight to Step 2. ```bash $(cat graphify-out/.graphify_python) -c " import json +import sys from graphify.detect import detect from pathlib import Path -result = detect(Path('INPUT_PATH')) +result = detect(Path(sys.argv[1])) print(json.dumps(result)) -" > graphify-out/.graphify_detect.json +" "INPUT_PATH" > graphify-out/.graphify_detect.json ``` Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: @@ -146,13 +147,11 @@ Skip this step entirely if `detect` returned zero `video` files. Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. -**Strategy:** Read the god nodes from the detect output or analysis file. You are already a language model - write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. - -**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` +**Strategy:** If an analysis file exists from a previous run, read its top god-node labels and write a one-sentence domain hint from them. Otherwise use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` Pass the chosen prompt to Whisper as its initial prompt. No separate API call needed. **Step 1 - Write the Whisper prompt yourself.** -Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: +Read the top god-node labels from previous analysis when available, then compose a short domain hint sentence. With no previous analysis, use the generic fallback above. - Labels: `transformer, attention, encoder, decoder` -> `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` - Labels: `kubernetes, deployment, pod, helm` -> `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` @@ -491,7 +490,7 @@ wrote = to_json(G, communities, 'graphify-out/graph.json') if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (fewer nodes than the existing graph). Run a full rebuild to be safe.') raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, sys.argv[1], suggested_questions=questions) Path('graphify-out/GRAPH_REPORT.md').write_text(report) analysis = { @@ -503,7 +502,7 @@ analysis = { } Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2)) print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" +" "INPUT_PATH" ``` If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. @@ -541,7 +540,7 @@ labels = LABELS_DICT # Regenerate questions with real community labels (labels affect question phrasing) questions = suggest_questions(G, communities, labels) -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, sys.argv[1], suggested_questions=questions) # Re-export so graph.json nodes carry the curated community_name (#2490). # Same extraction as Step 4, so the #479 shrink-guard passes on node count; # if it still refuses, surface the guard message - do not force past it. @@ -552,7 +551,7 @@ if not wrote: Path('graphify-out/GRAPH_REPORT.md').write_text(report) Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()})) print('Report updated with community labels') -" +" "INPUT_PATH" ``` Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). @@ -797,6 +796,7 @@ Print the output directly in chat. If `total_words <= 5000`, skip silently - the ```bash $(cat graphify-out/.graphify_python) -c " import json +import sys from pathlib import Path from datetime import datetime, timezone from graphify.detect import save_manifest @@ -807,13 +807,13 @@ extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) # Stamp only semantic files that produced output so a failed chunk is re-queued next run, not lost (#2015). from graphify.cli import _stamped_manifest_files _corpus = detect.get('all_files') or detect['files'] -_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) +_manifest_files = _stamped_manifest_files(_corpus, extract, Path(sys.argv[1])) _sem_types = ('document', 'paper', 'image') _dispatched = {f for t, fl in detect['files'].items() if t in _sem_types for f in fl} _stamped = {f for fl in _manifest_files.values() for f in fl} _cleared = _dispatched - _stamped _scan = {f for fl in _corpus.values() for f in fl} -save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) # Update cumulative cost tracker input_tok = extract.get('input_tokens', 0) @@ -837,7 +837,7 @@ cost_path.write_text(json.dumps(cost, indent=2)) print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" +" "INPUT_PATH" rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json graphify-out/.graphify_labels.json graphify-out/.graphify_incremental.json graphify-out/.graphify_transcripts.json graphify-out/.graphify_old.json; find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null rm -f graphify-out/needs_update 2>/dev/null || true ``` @@ -904,7 +904,7 @@ import sys, json from graphify.detect import detect_incremental, save_manifest from pathlib import Path -result = detect_incremental(Path('INPUT_PATH')) +result = detect_incremental(Path(sys.argv[1])) new_total = result.get('new_total', 0) print(json.dumps(result, indent=2)) Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result)) @@ -916,7 +916,7 @@ if deleted: print(f'{len(deleted)} deleted file(s) to prune.') if new_total > 0: print(f'{new_total} new/changed file(s) to re-extract.') -" +" "INPUT_PATH" ``` If new files exist, first check whether all changed files are code files: diff --git a/tools/skillgen/fragments/references/shared/transcribe.md b/tools/skillgen/fragments/references/shared/transcribe.md index b967f83799..bbe8b93e45 100644 --- a/tools/skillgen/fragments/references/shared/transcribe.md +++ b/tools/skillgen/fragments/references/shared/transcribe.md @@ -8,13 +8,11 @@ Skip this step entirely if `detect` returned zero `video` files. Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. -**Strategy:** Read the god nodes from `graphify-out/.graphify_detect.json` (or the analysis file if it exists from a previous run). You are already a language model — write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. - -**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` +**Strategy:** If `graphify-out/.graphify_analysis.json` exists from a previous run, read its top god-node labels and write a one-sentence domain hint from them. Otherwise use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` Pass the chosen prompt to Whisper as its initial prompt. No separate API call needed. **Step 1 - Write the Whisper prompt yourself.** -Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: +Read the top god-node labels from previous analysis when available, then compose a short domain hint sentence. With no previous analysis, use the generic fallback above. - Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` - Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` diff --git a/tools/skillgen/fragments/references/shared/update.md b/tools/skillgen/fragments/references/shared/update.md index bd8e659d3c..058ca8e6d2 100644 --- a/tools/skillgen/fragments/references/shared/update.md +++ b/tools/skillgen/fragments/references/shared/update.md @@ -12,7 +12,7 @@ import sys, json from graphify.detect import detect_incremental, save_manifest from pathlib import Path -result = detect_incremental(Path('INPUT_PATH')) +result = detect_incremental(Path(sys.argv[1])) new_total = result.get('new_total', 0) print(json.dumps(result, indent=2, ensure_ascii=False)) Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") @@ -24,7 +24,7 @@ if deleted: print(f'{len(deleted)} deleted file(s) to prune.') if new_total > 0: print(f'{new_total} new/changed file(s) to re-extract.') -" +" "INPUT_PATH" ``` Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: @@ -84,7 +84,7 @@ Then: ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from pathlib import Path from graphify.build import build_merge from graphify.detect import save_manifest @@ -113,7 +113,7 @@ G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, - root='INPUT_PATH', + root=sys.argv[1], directed=IS_DIRECTED, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -150,7 +150,7 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) # is lost forever (#2015). Mirrors the library extract path # (cli._stamped_manifest_files + clear_semantic + scan_corpus). from graphify.cli import _stamped_manifest_files -_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) +_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path(sys.argv[1])) # Changed semantic files dispatched this run but NOT stamped had their chunk fail # or be omitted; clear any stale semantic_hash so they are re-queued (#1948). _sem_types = ('document', 'paper', 'image') @@ -160,9 +160,9 @@ _cleared = _dispatched - _stamped # scan_corpus = the RAW full corpus so in-root files newly excluded since last run # are dropped rather than masquerading as deletions; untouched rows preserved (#1908). _scan = {f for fl in incremental['files'].values() for f in fl} -save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) print('[graphify update] Manifest saved.') -" +" "INPUT_PATH" ``` Then run Steps 4–8 on the merged graph as normal. diff --git a/tools/skillgen/gen.py b/tools/skillgen/gen.py index 6269d5dae1..afb3c826d3 100644 --- a/tools/skillgen/gen.py +++ b/tools/skillgen/gen.py @@ -384,6 +384,11 @@ def _render_frontmatter(platform: Platform) -> str: _PY_INVOKE_POSIX = '$(cat graphify-out/.graphify_python) -c "' _PY_INVOKE_PS_OPEN = "@'" _PY_INVOKE_PS_CLOSE = "'@ | & (Get-Content graphify-out\\.graphify_python) -" +_PY_CLOSE_TRANSLATIONS = { + '"': _PY_INVOKE_PS_CLOSE, + '" "INPUT_PATH"': _PY_INVOKE_PS_CLOSE + ' "INPUT_PATH"', + '" "INPUT_PATH" "SPEC_PATH"': _PY_INVOKE_PS_CLOSE + ' "INPUT_PATH" "SPEC_PATH"', +} _MKDIR_POSIX = "mkdir -p graphify-out" _MKDIR_PS = "New-Item -ItemType Directory -Force -Path graphify-out | Out-Null" _FIND_CHUNKS_POSIX = "find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null" @@ -434,8 +439,8 @@ def _translate_bash_block(lines: list[str]) -> list[str]: in_py = False for line in lines: if in_py: - if line == '"': - out.append(_PY_INVOKE_PS_CLOSE) + if line in _PY_CLOSE_TRANSLATIONS: + out.append(_PY_CLOSE_TRANSLATIONS[line]) in_py = False else: out.append(_unescape_bash_dq(line)) @@ -1169,6 +1174,38 @@ def _is_watch_path_quote_fix_line(line: str) -> bool: } +def _is_python_path_argument_fix_line(line: str) -> bool: + """Whether a monolith passes INPUT_PATH as argv instead of Python source.""" + return line.strip() in { + "import sys", + "result = detect(Path('INPUT_PATH'))", + "result = detect(Path(sys.argv[1]))", + "result = detect_incremental(Path('INPUT_PATH'))", + "result = detect_incremental(Path(sys.argv[1]))", + '"', + '" "INPUT_PATH"', + '" > .graphify_detect.json', + '" "INPUT_PATH" > .graphify_detect.json', + '" > graphify-out/.graphify_detect.json', + '" "INPUT_PATH" > graphify-out/.graphify_detect.json', + "report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions)", + "report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, sys.argv[1], suggested_questions=questions)", + "report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions)", + "report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, sys.argv[1], suggested_questions=questions)", + } + + +def _is_transcription_analysis_fix_line(line: str) -> bool: + """Whether transcription uses prior analysis rather than detect's nonexistent god nodes.""" + return line.strip() in { + '**Strategy:** Read the god nodes from the detect output or analysis file. You are already a language model - write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed.', + '**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."`', + 'Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example:', + '**Strategy:** If an analysis file exists from a previous run, read its top god-node labels and write a one-sentence domain hint from them. Otherwise use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` Pass the chosen prompt to Whisper as its initial prompt. No separate API call needed.', + 'Read the top god-node labels from previous analysis when available, then compose a short domain hint sentence. With no previous analysis, use the generic fallback above.', + } + + # Every line that may differ between a rendered monolith and its pristine v8 # baseline. Each predicate documents one sanctioned change-class; a blank line is # allowed because the multi-line fix blocks insert spacing. Anything else failing @@ -1193,6 +1230,8 @@ def _is_watch_path_quote_fix_line(line: str) -> bool: _is_needs_update_cleanup_fix_line, _is_graph_loader_fix_line, _is_watch_path_quote_fix_line, + _is_python_path_argument_fix_line, + _is_transcription_analysis_fix_line, ) From bc043e4f07f98d5dd26de59b8b0341a1679b6a01 Mon Sep 17 00:00:00 2001 From: Andre Brait Date: Thu, 10 Sep 2026 14:57:18 +0000 Subject: [PATCH 2/2] test: cover every generated path snippet --- tests/test_skillgen.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_skillgen.py b/tests/test_skillgen.py index 0c61152e1e..edf9c654a0 100644 --- a/tests/test_skillgen.py +++ b/tests/test_skillgen.py @@ -111,6 +111,8 @@ def test_rendered_instructions_preserve_scan_root_and_runnable_commands(): for artifact in watch_artifacts: assert 'graphify.watch "INPUT_PATH"' in artifact.content, artifact.path assert "graphify.watch INPUT_PATH" not in artifact.content, artifact.path + + for artifact in artifacts: assert "'INPUT_PATH'" not in artifact.content, artifact.path assert "'SPEC_PATH'" not in artifact.content, artifact.path