diff --git a/graphify/skill-agents.md b/graphify/skill-agents.md index 02be4d1f6f..190827d9ac 100644 --- a/graphify/skill-agents.md +++ b/graphify/skill-agents.md @@ -97,7 +97,7 @@ fi mkdir -p graphify-out "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" # Save scan root so `graphify update` (no args) knows where to look next time -echo "$(cd "INPUT_PATH" && pwd)" > graphify-out/.graphify_root +echo "$(cd INPUT_PATH && pwd)" > graphify-out/.graphify_root ``` If the import succeeds, print nothing and move straight to Step 2. @@ -108,15 +108,15 @@ If the import succeeds, print nothing and move straight to Step 2. ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from graphify.detect import detect from pathlib import Path -result = detect(Path(sys.argv[1])) +result = detect(Path('INPUT_PATH')) # Write the sidecar from Python, not a shell redirect, so the same block renders # on PowerShell hosts without console-encoding drift (#2528). Path('graphify-out/.graphify_detect.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") print(f'Detected {result[\"total_files\"]} files') -" "INPUT_PATH" +" ``` Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: @@ -176,6 +176,7 @@ $(cat graphify-out/.graphify_python) -c " import sys, json from graphify.extract import collect_files, extract from pathlib import Path +import json code_files = [] detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) @@ -183,13 +184,13 @@ for f in detect.get('files', {}).get('code', []): code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) if code_files: - result = extract(code_files, cache_root=Path(sys.argv[1])) + result = extract(code_files, cache_root=Path('INPUT_PATH')) Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') else: Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") print('No code files - skipping AST extraction') -" "INPUT_PATH" +" ``` #### Part B - Semantic extraction (parallel subagents) @@ -204,7 +205,7 @@ Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],' " ``` -**MANDATORY: You MUST use the subagent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the subagent tool you are doing this wrong.** +**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** Before dispatching subagents, print a timing estimate: - Load `total_words` and file counts from `graphify-out/.graphify_detect.json` @@ -220,7 +221,7 @@ SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from graphify.cache import check_semantic_cache from pathlib import Path @@ -230,7 +231,7 @@ detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encodin # every source file (#1392). Video is transcribed to a document in Step 2.5 first. all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root=sys.argv[1], prompt_file=sys.argv[2]) +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') # Always (re)write the cache file: write hits, else DELETE any leftover from a prior # run so Part C never merges a stale .graphify_cached.json (#1392). @@ -240,7 +241,7 @@ else: Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" "INPUT_PATH" "SPEC_PATH" +" ``` Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. @@ -282,7 +283,7 @@ Wait for all subagents. For each result: If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. -Merge all chunk files into `.graphify_semantic_new.json`. **After each subagent call completes, read the real token counts from the subagent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: +Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: ```bash $(cat graphify-out/.graphify_python) -c " import json, glob @@ -309,15 +310,15 @@ print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens' Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root=sys.argv[1], allowed_source_files=uncached, prompt_file=sys.argv[2]) +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') print(f'Cached {saved} files') -" "INPUT_PATH" "SPEC_PATH" +" ``` Merge cached + new results into `graphify-out/.graphify_semantic.json`: @@ -406,7 +407,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -431,7 +432,7 @@ if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') print('If this shrink is intentional (you deleted files), re-run a full build with --force.') raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, sys.argv[1], suggested_questions=questions) +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") analysis = { 'communities': {str(k): v for k, v in communities.items()}, @@ -442,7 +443,7 @@ analysis = { } Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" "INPUT_PATH" +" ``` If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. @@ -455,12 +456,12 @@ A non-destructive diagnostic on the extraction, before labeling. It surfaces edg ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from pathlib import Path from graphify.diagnostics import diagnose_extraction, format_diagnostic_report extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root=sys.argv[1]) +summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( ('dangling_endpoint_edges', 'dangling-endpoint edges'), @@ -470,7 +471,7 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0)] print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') -" "INPUT_PATH" +" ``` Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules). @@ -496,7 +497,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} @@ -507,7 +508,9 @@ labels = LABELS_DICT # Regenerate questions with real community labels (labels affect question phrasing) questions = suggest_questions(G, communities, labels) -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, sys.argv[1], suggested_questions=questions) +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") +Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") # Re-export so graph.json nodes carry the curated community_name (#2490). # Same extraction as Step 4, so the #479 shrink-guard passes on node count; # if it still refuses, surface the guard message - do not force past it. @@ -515,11 +518,8 @@ wrote = to_json(G, communities, 'graphify-out/graph.json', community_labels=labe if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') print('If this shrink is intentional (you deleted files), re-run a full build with --force.') - raise SystemExit(1) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") print('Report updated with community labels') -" "INPUT_PATH" +" ``` Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). @@ -534,14 +534,15 @@ If `--obsidian` was given: - If `--obsidian-dir ` was also given, pass it via `--dir`. Otherwise defaults to `graphify-out/obsidian`. ```bash -$(cat graphify-out/.graphify_python) -m graphify export obsidian -# Add `--dir ~/vaults/my-project` to use a custom directory. +graphify export obsidian +# or with custom dir: graphify export obsidian --dir ~/vaults/my-project ``` Generate the HTML graph (always, unless `--no-viz`): ```bash -$(cat graphify-out/.graphify_python) -m graphify export html # auto-aggregates to community view if graph > 5000 nodes +graphify export html # auto-aggregates to community view if graph > 5000 nodes +# or: graphify export html --no-viz ``` ### Steps 6b-8 - Wiki, Neo4j, FalkorDB, SVG, GraphML, MCP, benchmark (only on their flags) @@ -554,7 +555,7 @@ These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, ` ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from pathlib import Path from datetime import datetime, timezone from graphify.detect import save_manifest @@ -577,7 +578,7 @@ extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encod # types are gated on output. from graphify.cli import _stamped_manifest_files _corpus = detect.get('all_files') or detect['files'] -_manifest_files = _stamped_manifest_files(_corpus, extract, Path(sys.argv[1])) +_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) # Files dispatched this run (the changed subset) but NOT stamped above still carry # a stale semantic_hash from a prior run; clear it so detect_incremental re-queues # them instead of reading them as unchanged (#1948). @@ -589,7 +590,7 @@ _cleared = _dispatched - _stamped # files newly excluded since last run are dropped rather than masquerading as # deletions; untouched files' prior rows are still preserved (#1908). _scan = {f for fl in _corpus.values() for f in fl} -save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) # Update cumulative cost tracker input_tok = extract.get('input_tokens', 0) @@ -613,10 +614,10 @@ cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\" print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" "INPUT_PATH" +" rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null -rm -f graphify-out/needs_update 2>/dev/null || true +rm -f graphify-out/.needs_update 2>/dev/null || true ``` Replace INPUT_PATH with the actual path (same value used in Steps 4-5) so the manifest is relativized to the scan root. @@ -681,7 +682,7 @@ Both are non-default subcommands. `--update` re-extracts only new or changed fil When `graphify-out/graph.json` already exists and the user asks a question about the corpus, answer from the graph rather than rebuilding it: ```bash -$(cat graphify-out/.graphify_python) -m graphify query "" +graphify query "" ``` Before traversal, expand the question against the graph's own vocabulary so a wording mismatch does not collapse the answer to noise. If the `graphify query` CLI is unavailable, fall back to an inline NetworkX traversal of `graphify-out/graph.json`. Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. For that vocab-expansion step, the BFS/DFS traversal modes, the `--budget` cap, the NetworkX fallback, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`. diff --git a/graphify/skill-aider.md b/graphify/skill-aider.md index aacd9f31fc..4996beb787 100644 --- a/graphify/skill-aider.md +++ b/graphify/skill-aider.md @@ -101,12 +101,11 @@ If the import succeeds, print nothing and move straight to Step 2. ```bash $(cat graphify-out/.graphify_python) -c " import json -import sys from graphify.detect import detect from pathlib import Path -result = detect(Path(sys.argv[1])) +result = detect(Path('INPUT_PATH')) print(json.dumps(result)) -" "INPUT_PATH" > .graphify_detect.json +" > .graphify_detect.json ``` Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: @@ -134,11 +133,13 @@ Skip this step entirely if `detect` returned zero `video` files. Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. -**Strategy:** If an analysis file exists from a previous run, read its top god-node labels and write a one-sentence domain hint from them. Otherwise use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` Pass the chosen prompt to Whisper as its initial prompt. No separate API call needed. +**Strategy:** Read the god nodes from the detect output or analysis file. You are already a language model - write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. + +**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` **Step 1 - Write the Whisper prompt yourself.** -Read the top god-node labels from previous analysis when available, then compose a short domain hint sentence. With no previous analysis, use the generic fallback above. +Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: - Labels: `transformer, attention, encoder, decoder` -> `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` - Labels: `kubernetes, deployment, pod, helm` -> `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` @@ -425,7 +426,7 @@ wrote = to_json(G, communities, 'graphify-out/graph.json') if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (fewer nodes than the existing graph). Run a full rebuild to be safe.') raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, sys.argv[1], suggested_questions=questions) +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) Path('graphify-out/GRAPH_REPORT.md').write_text(report) analysis = { @@ -437,7 +438,7 @@ analysis = { } Path('.graphify_analysis.json').write_text(json.dumps(analysis, indent=2)) print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" "INPUT_PATH" +" ``` If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. @@ -475,18 +476,17 @@ labels = LABELS_DICT # Regenerate questions with real community labels (labels affect question phrasing) questions = suggest_questions(G, communities, labels) -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, sys.argv[1], suggested_questions=questions) +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report) +Path('.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()})) # Re-export so graph.json nodes carry the curated community_name (#2490). # Same extraction as Step 4, so the #479 shrink-guard passes on node count; # if it still refuses, surface the guard message - do not force past it. wrote = to_json(G, communities, 'graphify-out/graph.json', community_labels=labels) if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (fewer nodes than the existing graph). Run a full rebuild to be safe.') - raise SystemExit(1) -Path('graphify-out/GRAPH_REPORT.md').write_text(report) -Path('.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()})) print('Report updated with community labels') -" "INPUT_PATH" +" ``` Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). @@ -678,7 +678,6 @@ Print the output directly in chat. If `total_words <= 5000`, skip silently - the ```bash $(cat graphify-out/.graphify_python) -c " import json -import sys from pathlib import Path from datetime import datetime, timezone from graphify.detect import save_manifest @@ -689,13 +688,13 @@ extract = json.loads(Path('.graphify_extract.json').read_text()) # Stamp only semantic files that produced output so a failed chunk is re-queued next run, not lost (#2015). from graphify.cli import _stamped_manifest_files _corpus = detect.get('all_files') or detect['files'] -_manifest_files = _stamped_manifest_files(_corpus, extract, Path(sys.argv[1])) +_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) _sem_types = ('document', 'paper', 'image') _dispatched = {f for t, fl in detect['files'].items() if t in _sem_types for f in fl} _stamped = {f for fl in _manifest_files.values() for f in fl} _cleared = _dispatched - _stamped _scan = {f for fl in _corpus.values() for f in fl} -save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) # Update cumulative cost tracker input_tok = extract.get('input_tokens', 0) @@ -719,9 +718,9 @@ cost_path.write_text(json.dumps(cost, indent=2)) print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" "INPUT_PATH" +" rm -f .graphify_detect.json .graphify_extract.json .graphify_ast.json .graphify_semantic.json .graphify_analysis.json .graphify_labels.json; find . -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null -rm -f graphify-out/needs_update 2>/dev/null || true +rm -f graphify-out/.needs_update 2>/dev/null || true ``` Tell the user (omit the obsidian line unless --obsidian was given): @@ -765,7 +764,7 @@ import sys, json from graphify.detect import detect_incremental, save_manifest from pathlib import Path -result = detect_incremental(Path(sys.argv[1])) +result = detect_incremental(Path('INPUT_PATH')) new_total = result.get('new_total', 0) print(json.dumps(result, indent=2)) Path('.graphify_incremental.json').write_text(json.dumps(result)) @@ -777,7 +776,7 @@ if deleted: print(f'{len(deleted)} deleted file(s) to prune.') if new_total > 0: print(f'{new_total} new/changed file(s) to re-extract.') -" "INPUT_PATH" +" ``` If new files exist, first check whether all changed files are code files: @@ -822,13 +821,13 @@ $(cat graphify-out/.graphify_python) -c " import sys, json from graphify.build import build_from_json from graphify.export import to_json -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph import networkx as nx from pathlib import Path # Load existing graph existing_data = json.loads(Path('graphify-out/graph.json').read_text()) -G_existing = load_node_link_graph(existing_data) +G_existing = json_graph.node_link_graph(existing_data, edges='links') # Load new extraction new_extraction = json.loads(Path('.graphify_extract.json').read_text()) @@ -849,7 +848,7 @@ $(cat graphify-out/.graphify_python) -c " import json from graphify.analyze import graph_diff from graphify.build import build_from_json -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph import networkx as nx from pathlib import Path @@ -859,7 +858,7 @@ new_extract = json.loads(Path('.graphify_extract.json').read_text()) G_new = build_from_json(new_extract, directed=IS_DIRECTED) if old_data: - G_old = load_node_link_graph(old_data) + G_old = json_graph.node_link_graph(old_data, edges='links') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: @@ -885,12 +884,12 @@ from graphify.cluster import cluster, score_all from graphify.analyze import god_nodes, surprising_connections from graphify.report import generate from graphify.export import to_json -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph import networkx as nx from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text()) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') detection = {'total_files': 0, 'total_words': 99999, 'needs_graph': True, 'warning': None, 'files': {'code': [], 'document': [], 'paper': []}} @@ -954,12 +953,12 @@ Load `graphify-out/graph.json`, then: ```bash $(cat graphify-out/.graphify_python) -c " import sys, json -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph import networkx as nx from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text()) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' @@ -1069,11 +1068,11 @@ If it fails, stop and tell the user to run `/graphify ` first. $(cat graphify-out/.graphify_python) -c " import json, sys import networkx as nx -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text()) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') a_term = 'NODE_A' b_term = 'NODE_B' @@ -1142,11 +1141,11 @@ If it fails, stop and tell the user to run `/graphify ` first. $(cat graphify-out/.graphify_python) -c " import json, sys import networkx as nx -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text()) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') term = 'NODE_NAME' term_lower = term.lower() @@ -1200,7 +1199,7 @@ from graphify.ingest import ingest from pathlib import Path try: - out = ingest(sys.argv[1], Path('./raw'), author=sys.argv[2] or None, contributor=sys.argv[3] or None) + out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') print(f'Saved to {out}') except ValueError as e: print(f'error: {e}', file=sys.stderr) @@ -1208,10 +1207,10 @@ except ValueError as e: except RuntimeError as e: print(f'error: {e}', file=sys.stderr) sys.exit(1) -" "URL" "AUTHOR" "CONTRIBUTOR" +" ``` -Replace `URL` with the actual URL and pass the user's `AUTHOR` or `CONTRIBUTOR` when provided; use an empty string for either omitted value. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. +Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. Supported URL types (auto-detected): - Twitter/X → fetched via oEmbed, saved as `.md` with tweet text and author @@ -1227,7 +1226,7 @@ Supported URL types (auto-detected): Start a background watcher that monitors a folder and auto-updates the graph when files change. ```bash -python3 -m graphify.watch "INPUT_PATH" --debounce 3 +python3 -m graphify.watch INPUT_PATH --debounce 3 ``` Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: diff --git a/graphify/skill-amp.md b/graphify/skill-amp.md index 02be4d1f6f..190827d9ac 100644 --- a/graphify/skill-amp.md +++ b/graphify/skill-amp.md @@ -97,7 +97,7 @@ fi mkdir -p graphify-out "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" # Save scan root so `graphify update` (no args) knows where to look next time -echo "$(cd "INPUT_PATH" && pwd)" > graphify-out/.graphify_root +echo "$(cd INPUT_PATH && pwd)" > graphify-out/.graphify_root ``` If the import succeeds, print nothing and move straight to Step 2. @@ -108,15 +108,15 @@ If the import succeeds, print nothing and move straight to Step 2. ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from graphify.detect import detect from pathlib import Path -result = detect(Path(sys.argv[1])) +result = detect(Path('INPUT_PATH')) # Write the sidecar from Python, not a shell redirect, so the same block renders # on PowerShell hosts without console-encoding drift (#2528). Path('graphify-out/.graphify_detect.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") print(f'Detected {result[\"total_files\"]} files') -" "INPUT_PATH" +" ``` Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: @@ -176,6 +176,7 @@ $(cat graphify-out/.graphify_python) -c " import sys, json from graphify.extract import collect_files, extract from pathlib import Path +import json code_files = [] detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) @@ -183,13 +184,13 @@ for f in detect.get('files', {}).get('code', []): code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) if code_files: - result = extract(code_files, cache_root=Path(sys.argv[1])) + result = extract(code_files, cache_root=Path('INPUT_PATH')) Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') else: Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") print('No code files - skipping AST extraction') -" "INPUT_PATH" +" ``` #### Part B - Semantic extraction (parallel subagents) @@ -204,7 +205,7 @@ Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],' " ``` -**MANDATORY: You MUST use the subagent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the subagent tool you are doing this wrong.** +**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** Before dispatching subagents, print a timing estimate: - Load `total_words` and file counts from `graphify-out/.graphify_detect.json` @@ -220,7 +221,7 @@ SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from graphify.cache import check_semantic_cache from pathlib import Path @@ -230,7 +231,7 @@ detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encodin # every source file (#1392). Video is transcribed to a document in Step 2.5 first. all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root=sys.argv[1], prompt_file=sys.argv[2]) +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') # Always (re)write the cache file: write hits, else DELETE any leftover from a prior # run so Part C never merges a stale .graphify_cached.json (#1392). @@ -240,7 +241,7 @@ else: Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" "INPUT_PATH" "SPEC_PATH" +" ``` Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. @@ -282,7 +283,7 @@ Wait for all subagents. For each result: If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. -Merge all chunk files into `.graphify_semantic_new.json`. **After each subagent call completes, read the real token counts from the subagent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: +Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: ```bash $(cat graphify-out/.graphify_python) -c " import json, glob @@ -309,15 +310,15 @@ print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens' Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root=sys.argv[1], allowed_source_files=uncached, prompt_file=sys.argv[2]) +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') print(f'Cached {saved} files') -" "INPUT_PATH" "SPEC_PATH" +" ``` Merge cached + new results into `graphify-out/.graphify_semantic.json`: @@ -406,7 +407,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -431,7 +432,7 @@ if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') print('If this shrink is intentional (you deleted files), re-run a full build with --force.') raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, sys.argv[1], suggested_questions=questions) +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") analysis = { 'communities': {str(k): v for k, v in communities.items()}, @@ -442,7 +443,7 @@ analysis = { } Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" "INPUT_PATH" +" ``` If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. @@ -455,12 +456,12 @@ A non-destructive diagnostic on the extraction, before labeling. It surfaces edg ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from pathlib import Path from graphify.diagnostics import diagnose_extraction, format_diagnostic_report extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root=sys.argv[1]) +summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( ('dangling_endpoint_edges', 'dangling-endpoint edges'), @@ -470,7 +471,7 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0)] print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') -" "INPUT_PATH" +" ``` Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules). @@ -496,7 +497,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} @@ -507,7 +508,9 @@ labels = LABELS_DICT # Regenerate questions with real community labels (labels affect question phrasing) questions = suggest_questions(G, communities, labels) -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, sys.argv[1], suggested_questions=questions) +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") +Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") # Re-export so graph.json nodes carry the curated community_name (#2490). # Same extraction as Step 4, so the #479 shrink-guard passes on node count; # if it still refuses, surface the guard message - do not force past it. @@ -515,11 +518,8 @@ wrote = to_json(G, communities, 'graphify-out/graph.json', community_labels=labe if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') print('If this shrink is intentional (you deleted files), re-run a full build with --force.') - raise SystemExit(1) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") print('Report updated with community labels') -" "INPUT_PATH" +" ``` Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). @@ -534,14 +534,15 @@ If `--obsidian` was given: - If `--obsidian-dir ` was also given, pass it via `--dir`. Otherwise defaults to `graphify-out/obsidian`. ```bash -$(cat graphify-out/.graphify_python) -m graphify export obsidian -# Add `--dir ~/vaults/my-project` to use a custom directory. +graphify export obsidian +# or with custom dir: graphify export obsidian --dir ~/vaults/my-project ``` Generate the HTML graph (always, unless `--no-viz`): ```bash -$(cat graphify-out/.graphify_python) -m graphify export html # auto-aggregates to community view if graph > 5000 nodes +graphify export html # auto-aggregates to community view if graph > 5000 nodes +# or: graphify export html --no-viz ``` ### Steps 6b-8 - Wiki, Neo4j, FalkorDB, SVG, GraphML, MCP, benchmark (only on their flags) @@ -554,7 +555,7 @@ These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, ` ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from pathlib import Path from datetime import datetime, timezone from graphify.detect import save_manifest @@ -577,7 +578,7 @@ extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encod # types are gated on output. from graphify.cli import _stamped_manifest_files _corpus = detect.get('all_files') or detect['files'] -_manifest_files = _stamped_manifest_files(_corpus, extract, Path(sys.argv[1])) +_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) # Files dispatched this run (the changed subset) but NOT stamped above still carry # a stale semantic_hash from a prior run; clear it so detect_incremental re-queues # them instead of reading them as unchanged (#1948). @@ -589,7 +590,7 @@ _cleared = _dispatched - _stamped # files newly excluded since last run are dropped rather than masquerading as # deletions; untouched files' prior rows are still preserved (#1908). _scan = {f for fl in _corpus.values() for f in fl} -save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) # Update cumulative cost tracker input_tok = extract.get('input_tokens', 0) @@ -613,10 +614,10 @@ cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\" print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" "INPUT_PATH" +" rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null -rm -f graphify-out/needs_update 2>/dev/null || true +rm -f graphify-out/.needs_update 2>/dev/null || true ``` Replace INPUT_PATH with the actual path (same value used in Steps 4-5) so the manifest is relativized to the scan root. @@ -681,7 +682,7 @@ Both are non-default subcommands. `--update` re-extracts only new or changed fil When `graphify-out/graph.json` already exists and the user asks a question about the corpus, answer from the graph rather than rebuilding it: ```bash -$(cat graphify-out/.graphify_python) -m graphify query "" +graphify query "" ``` Before traversal, expand the question against the graph's own vocabulary so a wording mismatch does not collapse the answer to noise. If the `graphify query` CLI is unavailable, fall back to an inline NetworkX traversal of `graphify-out/graph.json`. Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. For that vocab-expansion step, the BFS/DFS traversal modes, the `--budget` cap, the NetworkX fallback, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`. diff --git a/graphify/skill-claw.md b/graphify/skill-claw.md index 55b77d6828..abd2811d23 100644 --- a/graphify/skill-claw.md +++ b/graphify/skill-claw.md @@ -97,7 +97,7 @@ fi mkdir -p graphify-out "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" # Save scan root so `graphify update` (no args) knows where to look next time -echo "$(cd "INPUT_PATH" && pwd)" > graphify-out/.graphify_root +echo "$(cd INPUT_PATH && pwd)" > graphify-out/.graphify_root ``` If the import succeeds, print nothing and move straight to Step 2. @@ -108,15 +108,15 @@ If the import succeeds, print nothing and move straight to Step 2. ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from graphify.detect import detect from pathlib import Path -result = detect(Path(sys.argv[1])) +result = detect(Path('INPUT_PATH')) # Write the sidecar from Python, not a shell redirect, so the same block renders # on PowerShell hosts without console-encoding drift (#2528). Path('graphify-out/.graphify_detect.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") print(f'Detected {result[\"total_files\"]} files') -" "INPUT_PATH" +" ``` Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: @@ -176,6 +176,7 @@ $(cat graphify-out/.graphify_python) -c " import sys, json from graphify.extract import collect_files, extract from pathlib import Path +import json code_files = [] detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) @@ -183,13 +184,13 @@ for f in detect.get('files', {}).get('code', []): code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) if code_files: - result = extract(code_files, cache_root=Path(sys.argv[1])) + result = extract(code_files, cache_root=Path('INPUT_PATH')) Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') else: Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") print('No code files - skipping AST extraction') -" "INPUT_PATH" +" ``` #### Part B - Semantic extraction (parallel subagents) @@ -204,7 +205,7 @@ Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],' " ``` -**MANDATORY: You MUST use the subagent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the subagent tool you are doing this wrong.** +**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** Before dispatching subagents, print a timing estimate: - Load `total_words` and file counts from `graphify-out/.graphify_detect.json` @@ -220,7 +221,7 @@ SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from graphify.cache import check_semantic_cache from pathlib import Path @@ -230,7 +231,7 @@ detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encodin # every source file (#1392). Video is transcribed to a document in Step 2.5 first. all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root=sys.argv[1], prompt_file=sys.argv[2]) +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') # Always (re)write the cache file: write hits, else DELETE any leftover from a prior # run so Part C never merges a stale .graphify_cached.json (#1392). @@ -240,7 +241,7 @@ else: Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" "INPUT_PATH" "SPEC_PATH" +" ``` Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. @@ -285,7 +286,7 @@ Wait for all subagents. For each result: If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. -Merge all chunk files into `.graphify_semantic_new.json`. **After each subagent call completes, read the real token counts from the subagent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: +Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: ```bash $(cat graphify-out/.graphify_python) -c " import json, glob @@ -312,15 +313,15 @@ print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens' Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root=sys.argv[1], allowed_source_files=uncached, prompt_file=sys.argv[2]) +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') print(f'Cached {saved} files') -" "INPUT_PATH" "SPEC_PATH" +" ``` Merge cached + new results into `graphify-out/.graphify_semantic.json`: @@ -409,7 +410,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -434,7 +435,7 @@ if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') print('If this shrink is intentional (you deleted files), re-run a full build with --force.') raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, sys.argv[1], suggested_questions=questions) +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") analysis = { 'communities': {str(k): v for k, v in communities.items()}, @@ -445,7 +446,7 @@ analysis = { } Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" "INPUT_PATH" +" ``` If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. @@ -458,12 +459,12 @@ A non-destructive diagnostic on the extraction, before labeling. It surfaces edg ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from pathlib import Path from graphify.diagnostics import diagnose_extraction, format_diagnostic_report extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root=sys.argv[1]) +summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( ('dangling_endpoint_edges', 'dangling-endpoint edges'), @@ -473,7 +474,7 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0)] print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') -" "INPUT_PATH" +" ``` Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules). @@ -499,7 +500,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} @@ -510,7 +511,9 @@ labels = LABELS_DICT # Regenerate questions with real community labels (labels affect question phrasing) questions = suggest_questions(G, communities, labels) -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, sys.argv[1], suggested_questions=questions) +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") +Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") # Re-export so graph.json nodes carry the curated community_name (#2490). # Same extraction as Step 4, so the #479 shrink-guard passes on node count; # if it still refuses, surface the guard message - do not force past it. @@ -518,11 +521,8 @@ wrote = to_json(G, communities, 'graphify-out/graph.json', community_labels=labe if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') print('If this shrink is intentional (you deleted files), re-run a full build with --force.') - raise SystemExit(1) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") print('Report updated with community labels') -" "INPUT_PATH" +" ``` Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). @@ -537,14 +537,15 @@ If `--obsidian` was given: - If `--obsidian-dir ` was also given, pass it via `--dir`. Otherwise defaults to `graphify-out/obsidian`. ```bash -$(cat graphify-out/.graphify_python) -m graphify export obsidian -# Add `--dir ~/vaults/my-project` to use a custom directory. +graphify export obsidian +# or with custom dir: graphify export obsidian --dir ~/vaults/my-project ``` Generate the HTML graph (always, unless `--no-viz`): ```bash -$(cat graphify-out/.graphify_python) -m graphify export html # auto-aggregates to community view if graph > 5000 nodes +graphify export html # auto-aggregates to community view if graph > 5000 nodes +# or: graphify export html --no-viz ``` ### Steps 6b-8 - Wiki, Neo4j, FalkorDB, SVG, GraphML, MCP, benchmark (only on their flags) @@ -557,7 +558,7 @@ These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, ` ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from pathlib import Path from datetime import datetime, timezone from graphify.detect import save_manifest @@ -580,7 +581,7 @@ extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encod # types are gated on output. from graphify.cli import _stamped_manifest_files _corpus = detect.get('all_files') or detect['files'] -_manifest_files = _stamped_manifest_files(_corpus, extract, Path(sys.argv[1])) +_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) # Files dispatched this run (the changed subset) but NOT stamped above still carry # a stale semantic_hash from a prior run; clear it so detect_incremental re-queues # them instead of reading them as unchanged (#1948). @@ -592,7 +593,7 @@ _cleared = _dispatched - _stamped # files newly excluded since last run are dropped rather than masquerading as # deletions; untouched files' prior rows are still preserved (#1908). _scan = {f for fl in _corpus.values() for f in fl} -save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) # Update cumulative cost tracker input_tok = extract.get('input_tokens', 0) @@ -616,10 +617,10 @@ cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\" print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" "INPUT_PATH" +" rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null -rm -f graphify-out/needs_update 2>/dev/null || true +rm -f graphify-out/.needs_update 2>/dev/null || true ``` Replace INPUT_PATH with the actual path (same value used in Steps 4-5) so the manifest is relativized to the scan root. @@ -684,7 +685,7 @@ Both are non-default subcommands. `--update` re-extracts only new or changed fil When `graphify-out/graph.json` already exists and the user asks a question about the corpus, answer from the graph rather than rebuilding it: ```bash -$(cat graphify-out/.graphify_python) -m graphify query "" +graphify query "" ``` Before traversal, expand the question against the graph's own vocabulary so a wording mismatch does not collapse the answer to noise. If the `graphify query` CLI is unavailable, fall back to an inline NetworkX traversal of `graphify-out/graph.json`. Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. For that vocab-expansion step, the BFS/DFS traversal modes, the `--budget` cap, the NetworkX fallback, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`. diff --git a/graphify/skill-codex.md b/graphify/skill-codex.md index fbd068db2b..af3f723c78 100644 --- a/graphify/skill-codex.md +++ b/graphify/skill-codex.md @@ -97,7 +97,7 @@ fi mkdir -p graphify-out "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" # Save scan root so `graphify update` (no args) knows where to look next time -echo "$(cd "INPUT_PATH" && pwd)" > graphify-out/.graphify_root +echo "$(cd INPUT_PATH && pwd)" > graphify-out/.graphify_root ``` If the import succeeds, print nothing and move straight to Step 2. @@ -108,15 +108,15 @@ If the import succeeds, print nothing and move straight to Step 2. ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from graphify.detect import detect from pathlib import Path -result = detect(Path(sys.argv[1])) +result = detect(Path('INPUT_PATH')) # Write the sidecar from Python, not a shell redirect, so the same block renders # on PowerShell hosts without console-encoding drift (#2528). Path('graphify-out/.graphify_detect.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") print(f'Detected {result[\"total_files\"]} files') -" "INPUT_PATH" +" ``` Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: @@ -176,6 +176,7 @@ $(cat graphify-out/.graphify_python) -c " import sys, json from graphify.extract import collect_files, extract from pathlib import Path +import json code_files = [] detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) @@ -183,13 +184,13 @@ for f in detect.get('files', {}).get('code', []): code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) if code_files: - result = extract(code_files, cache_root=Path(sys.argv[1])) + result = extract(code_files, cache_root=Path('INPUT_PATH')) Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') else: Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") print('No code files - skipping AST extraction') -" "INPUT_PATH" +" ``` #### Part B - Semantic extraction (parallel subagents) @@ -204,7 +205,7 @@ Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],' " ``` -**MANDATORY: You MUST use the subagent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the subagent tool you are doing this wrong.** +**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** Before dispatching subagents, print a timing estimate: - Load `total_words` and file counts from `graphify-out/.graphify_detect.json` @@ -220,7 +221,7 @@ SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from graphify.cache import check_semantic_cache from pathlib import Path @@ -230,7 +231,7 @@ detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encodin # every source file (#1392). Video is transcribed to a document in Step 2.5 first. all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root=sys.argv[1], prompt_file=sys.argv[2]) +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') # Always (re)write the cache file: write hits, else DELETE any leftover from a prior # run so Part C never merges a stale .graphify_cached.json (#1392). @@ -240,7 +241,7 @@ else: Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" "INPUT_PATH" "SPEC_PATH" +" ``` Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. @@ -282,7 +283,7 @@ Wait for all subagents. For each result: If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. -Merge all chunk files into `.graphify_semantic_new.json`. **After each subagent call completes, read the real token counts from the subagent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: +Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: ```bash $(cat graphify-out/.graphify_python) -c " import json, glob @@ -309,15 +310,15 @@ print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens' Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root=sys.argv[1], allowed_source_files=uncached, prompt_file=sys.argv[2]) +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') print(f'Cached {saved} files') -" "INPUT_PATH" "SPEC_PATH" +" ``` Merge cached + new results into `graphify-out/.graphify_semantic.json`: @@ -406,7 +407,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -431,7 +432,7 @@ if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') print('If this shrink is intentional (you deleted files), re-run a full build with --force.') raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, sys.argv[1], suggested_questions=questions) +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") analysis = { 'communities': {str(k): v for k, v in communities.items()}, @@ -442,7 +443,7 @@ analysis = { } Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" "INPUT_PATH" +" ``` If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. @@ -455,12 +456,12 @@ A non-destructive diagnostic on the extraction, before labeling. It surfaces edg ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from pathlib import Path from graphify.diagnostics import diagnose_extraction, format_diagnostic_report extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root=sys.argv[1]) +summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( ('dangling_endpoint_edges', 'dangling-endpoint edges'), @@ -470,7 +471,7 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0)] print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') -" "INPUT_PATH" +" ``` Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules). @@ -496,7 +497,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} @@ -507,7 +508,9 @@ labels = LABELS_DICT # Regenerate questions with real community labels (labels affect question phrasing) questions = suggest_questions(G, communities, labels) -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, sys.argv[1], suggested_questions=questions) +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") +Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") # Re-export so graph.json nodes carry the curated community_name (#2490). # Same extraction as Step 4, so the #479 shrink-guard passes on node count; # if it still refuses, surface the guard message - do not force past it. @@ -515,11 +518,8 @@ wrote = to_json(G, communities, 'graphify-out/graph.json', community_labels=labe if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') print('If this shrink is intentional (you deleted files), re-run a full build with --force.') - raise SystemExit(1) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") print('Report updated with community labels') -" "INPUT_PATH" +" ``` Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). @@ -534,14 +534,15 @@ If `--obsidian` was given: - If `--obsidian-dir ` was also given, pass it via `--dir`. Otherwise defaults to `graphify-out/obsidian`. ```bash -$(cat graphify-out/.graphify_python) -m graphify export obsidian -# Add `--dir ~/vaults/my-project` to use a custom directory. +graphify export obsidian +# or with custom dir: graphify export obsidian --dir ~/vaults/my-project ``` Generate the HTML graph (always, unless `--no-viz`): ```bash -$(cat graphify-out/.graphify_python) -m graphify export html # auto-aggregates to community view if graph > 5000 nodes +graphify export html # auto-aggregates to community view if graph > 5000 nodes +# or: graphify export html --no-viz ``` ### Steps 6b-8 - Wiki, Neo4j, FalkorDB, SVG, GraphML, MCP, benchmark (only on their flags) @@ -554,7 +555,7 @@ These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, ` ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from pathlib import Path from datetime import datetime, timezone from graphify.detect import save_manifest @@ -577,7 +578,7 @@ extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encod # types are gated on output. from graphify.cli import _stamped_manifest_files _corpus = detect.get('all_files') or detect['files'] -_manifest_files = _stamped_manifest_files(_corpus, extract, Path(sys.argv[1])) +_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) # Files dispatched this run (the changed subset) but NOT stamped above still carry # a stale semantic_hash from a prior run; clear it so detect_incremental re-queues # them instead of reading them as unchanged (#1948). @@ -589,7 +590,7 @@ _cleared = _dispatched - _stamped # files newly excluded since last run are dropped rather than masquerading as # deletions; untouched files' prior rows are still preserved (#1908). _scan = {f for fl in _corpus.values() for f in fl} -save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) # Update cumulative cost tracker input_tok = extract.get('input_tokens', 0) @@ -613,10 +614,10 @@ cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\" print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" "INPUT_PATH" +" rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null -rm -f graphify-out/needs_update 2>/dev/null || true +rm -f graphify-out/.needs_update 2>/dev/null || true ``` Replace INPUT_PATH with the actual path (same value used in Steps 4-5) so the manifest is relativized to the scan root. @@ -681,7 +682,7 @@ Both are non-default subcommands. `--update` re-extracts only new or changed fil When `graphify-out/graph.json` already exists and the user asks a question about the corpus, answer from the graph rather than rebuilding it: ```bash -$(cat graphify-out/.graphify_python) -m graphify query "" +graphify query "" ``` Before traversal, expand the question against the graph's own vocabulary so a wording mismatch does not collapse the answer to noise. If the `graphify query` CLI is unavailable, fall back to an inline NetworkX traversal of `graphify-out/graph.json`. Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. For that vocab-expansion step, the BFS/DFS traversal modes, the `--budget` cap, the NetworkX fallback, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`. diff --git a/graphify/skill-copilot.md b/graphify/skill-copilot.md index 55b77d6828..abd2811d23 100644 --- a/graphify/skill-copilot.md +++ b/graphify/skill-copilot.md @@ -97,7 +97,7 @@ fi mkdir -p graphify-out "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" # Save scan root so `graphify update` (no args) knows where to look next time -echo "$(cd "INPUT_PATH" && pwd)" > graphify-out/.graphify_root +echo "$(cd INPUT_PATH && pwd)" > graphify-out/.graphify_root ``` If the import succeeds, print nothing and move straight to Step 2. @@ -108,15 +108,15 @@ If the import succeeds, print nothing and move straight to Step 2. ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from graphify.detect import detect from pathlib import Path -result = detect(Path(sys.argv[1])) +result = detect(Path('INPUT_PATH')) # Write the sidecar from Python, not a shell redirect, so the same block renders # on PowerShell hosts without console-encoding drift (#2528). Path('graphify-out/.graphify_detect.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") print(f'Detected {result[\"total_files\"]} files') -" "INPUT_PATH" +" ``` Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: @@ -176,6 +176,7 @@ $(cat graphify-out/.graphify_python) -c " import sys, json from graphify.extract import collect_files, extract from pathlib import Path +import json code_files = [] detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) @@ -183,13 +184,13 @@ for f in detect.get('files', {}).get('code', []): code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) if code_files: - result = extract(code_files, cache_root=Path(sys.argv[1])) + result = extract(code_files, cache_root=Path('INPUT_PATH')) Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') else: Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") print('No code files - skipping AST extraction') -" "INPUT_PATH" +" ``` #### Part B - Semantic extraction (parallel subagents) @@ -204,7 +205,7 @@ Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],' " ``` -**MANDATORY: You MUST use the subagent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the subagent tool you are doing this wrong.** +**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** Before dispatching subagents, print a timing estimate: - Load `total_words` and file counts from `graphify-out/.graphify_detect.json` @@ -220,7 +221,7 @@ SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from graphify.cache import check_semantic_cache from pathlib import Path @@ -230,7 +231,7 @@ detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encodin # every source file (#1392). Video is transcribed to a document in Step 2.5 first. all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root=sys.argv[1], prompt_file=sys.argv[2]) +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') # Always (re)write the cache file: write hits, else DELETE any leftover from a prior # run so Part C never merges a stale .graphify_cached.json (#1392). @@ -240,7 +241,7 @@ else: Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" "INPUT_PATH" "SPEC_PATH" +" ``` Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. @@ -285,7 +286,7 @@ Wait for all subagents. For each result: If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. -Merge all chunk files into `.graphify_semantic_new.json`. **After each subagent call completes, read the real token counts from the subagent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: +Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: ```bash $(cat graphify-out/.graphify_python) -c " import json, glob @@ -312,15 +313,15 @@ print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens' Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root=sys.argv[1], allowed_source_files=uncached, prompt_file=sys.argv[2]) +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') print(f'Cached {saved} files') -" "INPUT_PATH" "SPEC_PATH" +" ``` Merge cached + new results into `graphify-out/.graphify_semantic.json`: @@ -409,7 +410,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -434,7 +435,7 @@ if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') print('If this shrink is intentional (you deleted files), re-run a full build with --force.') raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, sys.argv[1], suggested_questions=questions) +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") analysis = { 'communities': {str(k): v for k, v in communities.items()}, @@ -445,7 +446,7 @@ analysis = { } Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" "INPUT_PATH" +" ``` If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. @@ -458,12 +459,12 @@ A non-destructive diagnostic on the extraction, before labeling. It surfaces edg ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from pathlib import Path from graphify.diagnostics import diagnose_extraction, format_diagnostic_report extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root=sys.argv[1]) +summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( ('dangling_endpoint_edges', 'dangling-endpoint edges'), @@ -473,7 +474,7 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0)] print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') -" "INPUT_PATH" +" ``` Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules). @@ -499,7 +500,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} @@ -510,7 +511,9 @@ labels = LABELS_DICT # Regenerate questions with real community labels (labels affect question phrasing) questions = suggest_questions(G, communities, labels) -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, sys.argv[1], suggested_questions=questions) +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") +Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") # Re-export so graph.json nodes carry the curated community_name (#2490). # Same extraction as Step 4, so the #479 shrink-guard passes on node count; # if it still refuses, surface the guard message - do not force past it. @@ -518,11 +521,8 @@ wrote = to_json(G, communities, 'graphify-out/graph.json', community_labels=labe if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') print('If this shrink is intentional (you deleted files), re-run a full build with --force.') - raise SystemExit(1) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") print('Report updated with community labels') -" "INPUT_PATH" +" ``` Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). @@ -537,14 +537,15 @@ If `--obsidian` was given: - If `--obsidian-dir ` was also given, pass it via `--dir`. Otherwise defaults to `graphify-out/obsidian`. ```bash -$(cat graphify-out/.graphify_python) -m graphify export obsidian -# Add `--dir ~/vaults/my-project` to use a custom directory. +graphify export obsidian +# or with custom dir: graphify export obsidian --dir ~/vaults/my-project ``` Generate the HTML graph (always, unless `--no-viz`): ```bash -$(cat graphify-out/.graphify_python) -m graphify export html # auto-aggregates to community view if graph > 5000 nodes +graphify export html # auto-aggregates to community view if graph > 5000 nodes +# or: graphify export html --no-viz ``` ### Steps 6b-8 - Wiki, Neo4j, FalkorDB, SVG, GraphML, MCP, benchmark (only on their flags) @@ -557,7 +558,7 @@ These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, ` ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from pathlib import Path from datetime import datetime, timezone from graphify.detect import save_manifest @@ -580,7 +581,7 @@ extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encod # types are gated on output. from graphify.cli import _stamped_manifest_files _corpus = detect.get('all_files') or detect['files'] -_manifest_files = _stamped_manifest_files(_corpus, extract, Path(sys.argv[1])) +_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) # Files dispatched this run (the changed subset) but NOT stamped above still carry # a stale semantic_hash from a prior run; clear it so detect_incremental re-queues # them instead of reading them as unchanged (#1948). @@ -592,7 +593,7 @@ _cleared = _dispatched - _stamped # files newly excluded since last run are dropped rather than masquerading as # deletions; untouched files' prior rows are still preserved (#1908). _scan = {f for fl in _corpus.values() for f in fl} -save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) # Update cumulative cost tracker input_tok = extract.get('input_tokens', 0) @@ -616,10 +617,10 @@ cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\" print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" "INPUT_PATH" +" rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null -rm -f graphify-out/needs_update 2>/dev/null || true +rm -f graphify-out/.needs_update 2>/dev/null || true ``` Replace INPUT_PATH with the actual path (same value used in Steps 4-5) so the manifest is relativized to the scan root. @@ -684,7 +685,7 @@ Both are non-default subcommands. `--update` re-extracts only new or changed fil When `graphify-out/graph.json` already exists and the user asks a question about the corpus, answer from the graph rather than rebuilding it: ```bash -$(cat graphify-out/.graphify_python) -m graphify query "" +graphify query "" ``` Before traversal, expand the question against the graph's own vocabulary so a wording mismatch does not collapse the answer to noise. If the `graphify query` CLI is unavailable, fall back to an inline NetworkX traversal of `graphify-out/graph.json`. Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. For that vocab-expansion step, the BFS/DFS traversal modes, the `--budget` cap, the NetworkX fallback, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`. diff --git a/graphify/skill-devin.md b/graphify/skill-devin.md index 8740ca565a..f9be846cbf 100644 --- a/graphify/skill-devin.md +++ b/graphify/skill-devin.md @@ -114,12 +114,11 @@ If the import succeeds, print nothing and move straight to Step 2. ```bash $(cat graphify-out/.graphify_python) -c " import json -import sys from graphify.detect import detect from pathlib import Path -result = detect(Path(sys.argv[1])) +result = detect(Path('INPUT_PATH')) print(json.dumps(result)) -" "INPUT_PATH" > graphify-out/.graphify_detect.json +" > graphify-out/.graphify_detect.json ``` Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: @@ -147,11 +146,13 @@ Skip this step entirely if `detect` returned zero `video` files. Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. -**Strategy:** If an analysis file exists from a previous run, read its top god-node labels and write a one-sentence domain hint from them. Otherwise use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` Pass the chosen prompt to Whisper as its initial prompt. No separate API call needed. +**Strategy:** Read the god nodes from the detect output or analysis file. You are already a language model - write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. + +**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` **Step 1 - Write the Whisper prompt yourself.** -Read the top god-node labels from previous analysis when available, then compose a short domain hint sentence. With no previous analysis, use the generic fallback above. +Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: - Labels: `transformer, attention, encoder, decoder` -> `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` - Labels: `kubernetes, deployment, pod, helm` -> `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` @@ -490,7 +491,7 @@ wrote = to_json(G, communities, 'graphify-out/graph.json') if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (fewer nodes than the existing graph). Run a full rebuild to be safe.') raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, sys.argv[1], suggested_questions=questions) +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) Path('graphify-out/GRAPH_REPORT.md').write_text(report) analysis = { @@ -502,7 +503,7 @@ analysis = { } Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2)) print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" "INPUT_PATH" +" ``` If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. @@ -540,18 +541,17 @@ labels = LABELS_DICT # Regenerate questions with real community labels (labels affect question phrasing) questions = suggest_questions(G, communities, labels) -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, sys.argv[1], suggested_questions=questions) +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report) +Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()})) # Re-export so graph.json nodes carry the curated community_name (#2490). # Same extraction as Step 4, so the #479 shrink-guard passes on node count; # if it still refuses, surface the guard message - do not force past it. wrote = to_json(G, communities, 'graphify-out/graph.json', community_labels=labels) if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (fewer nodes than the existing graph). Run a full rebuild to be safe.') - raise SystemExit(1) -Path('graphify-out/GRAPH_REPORT.md').write_text(report) -Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()})) print('Report updated with community labels') -" "INPUT_PATH" +" ``` Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). @@ -796,7 +796,6 @@ Print the output directly in chat. If `total_words <= 5000`, skip silently - the ```bash $(cat graphify-out/.graphify_python) -c " import json -import sys from pathlib import Path from datetime import datetime, timezone from graphify.detect import save_manifest @@ -807,13 +806,13 @@ extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) # Stamp only semantic files that produced output so a failed chunk is re-queued next run, not lost (#2015). from graphify.cli import _stamped_manifest_files _corpus = detect.get('all_files') or detect['files'] -_manifest_files = _stamped_manifest_files(_corpus, extract, Path(sys.argv[1])) +_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) _sem_types = ('document', 'paper', 'image') _dispatched = {f for t, fl in detect['files'].items() if t in _sem_types for f in fl} _stamped = {f for fl in _manifest_files.values() for f in fl} _cleared = _dispatched - _stamped _scan = {f for fl in _corpus.values() for f in fl} -save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) # Update cumulative cost tracker input_tok = extract.get('input_tokens', 0) @@ -837,9 +836,9 @@ cost_path.write_text(json.dumps(cost, indent=2)) print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" "INPUT_PATH" +" rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json graphify-out/.graphify_labels.json graphify-out/.graphify_incremental.json graphify-out/.graphify_transcripts.json graphify-out/.graphify_old.json; find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null -rm -f graphify-out/needs_update 2>/dev/null || true +rm -f graphify-out/.needs_update 2>/dev/null || true ``` Tell the user (omit the obsidian line unless --obsidian was given; omit the wiki line unless --wiki was given): @@ -904,7 +903,7 @@ import sys, json from graphify.detect import detect_incremental, save_manifest from pathlib import Path -result = detect_incremental(Path(sys.argv[1])) +result = detect_incremental(Path('INPUT_PATH')) new_total = result.get('new_total', 0) print(json.dumps(result, indent=2)) Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result)) @@ -916,7 +915,7 @@ if deleted: print(f'{len(deleted)} deleted file(s) to prune.') if new_total > 0: print(f'{new_total} new/changed file(s) to re-extract.') -" "INPUT_PATH" +" ``` If new files exist, first check whether all changed files are code files: @@ -959,13 +958,13 @@ $(cat graphify-out/.graphify_python) -c " import sys, json from graphify.build import build_from_json from graphify.export import to_json -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph import networkx as nx from pathlib import Path # Load existing graph existing_data = json.loads(Path('graphify-out/graph.json').read_text()) -G_existing = load_node_link_graph(existing_data) +G_existing = json_graph.node_link_graph(existing_data, edges='links') # Load new extraction new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) @@ -986,7 +985,7 @@ $(cat graphify-out/.graphify_python) -c " import json from graphify.analyze import graph_diff from graphify.build import build_from_json -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph import networkx as nx from pathlib import Path @@ -995,7 +994,7 @@ new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text() G_new = build_from_json(new_extract, directed=IS_DIRECTED) if old_data: - G_old = load_node_link_graph(old_data) + G_old = json_graph.node_link_graph(old_data, edges='links') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: @@ -1021,12 +1020,12 @@ from graphify.cluster import cluster, score_all from graphify.analyze import god_nodes, surprising_connections from graphify.report import generate from graphify.export import to_json -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph import networkx as nx from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text()) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') detection = {'total_files': 0, 'total_words': 99999, 'needs_graph': True, 'warning': None, 'files': {'code': [], 'document': [], 'paper': []}} @@ -1090,12 +1089,12 @@ Load `graphify-out/graph.json`, then: ```bash $(cat graphify-out/.graphify_python) -c " import sys, json -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph import networkx as nx from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text()) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' @@ -1202,11 +1201,11 @@ if not Path('graphify-out/graph.json').exists(): $(cat graphify-out/.graphify_python) -c " import json, sys import networkx as nx -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text()) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') a_term = 'NODE_A' b_term = 'NODE_B' @@ -1274,11 +1273,11 @@ if not Path('graphify-out/graph.json').exists(): $(cat graphify-out/.graphify_python) -c " import json, sys import networkx as nx -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text()) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') term = 'NODE_NAME' term_lower = term.lower() @@ -1332,7 +1331,7 @@ from graphify.ingest import ingest from pathlib import Path try: - out = ingest(sys.argv[1], Path('./raw'), author=sys.argv[2] or None, contributor=sys.argv[3] or None) + out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') print(f'Saved to {out}') except ValueError as e: print(f'error: {e}', file=sys.stderr) @@ -1340,10 +1339,10 @@ except ValueError as e: except RuntimeError as e: print(f'error: {e}', file=sys.stderr) sys.exit(1) -" "URL" "AUTHOR" "CONTRIBUTOR" +" ``` -Replace `URL` with the actual URL and pass the user's `AUTHOR` or `CONTRIBUTOR` when provided; use an empty string for either omitted value. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. +Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. Supported URL types (auto-detected): - Twitter/X -> fetched via oEmbed, saved as `.md` with tweet text and author @@ -1359,7 +1358,7 @@ Supported URL types (auto-detected): Start a background watcher that monitors a folder and auto-updates the graph when files change. ```bash -python3 -m graphify.watch "INPUT_PATH" --debounce 3 +python3 -m graphify.watch INPUT_PATH --debounce 3 ``` Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: diff --git a/graphify/skill-droid.md b/graphify/skill-droid.md index b9de11f80c..fd148d485d 100644 --- a/graphify/skill-droid.md +++ b/graphify/skill-droid.md @@ -97,7 +97,7 @@ fi mkdir -p graphify-out "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" # Save scan root so `graphify update` (no args) knows where to look next time -echo "$(cd "INPUT_PATH" && pwd)" > graphify-out/.graphify_root +echo "$(cd INPUT_PATH && pwd)" > graphify-out/.graphify_root ``` If the import succeeds, print nothing and move straight to Step 2. @@ -108,15 +108,15 @@ If the import succeeds, print nothing and move straight to Step 2. ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from graphify.detect import detect from pathlib import Path -result = detect(Path(sys.argv[1])) +result = detect(Path('INPUT_PATH')) # Write the sidecar from Python, not a shell redirect, so the same block renders # on PowerShell hosts without console-encoding drift (#2528). Path('graphify-out/.graphify_detect.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") print(f'Detected {result[\"total_files\"]} files') -" "INPUT_PATH" +" ``` Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: @@ -176,6 +176,7 @@ $(cat graphify-out/.graphify_python) -c " import sys, json from graphify.extract import collect_files, extract from pathlib import Path +import json code_files = [] detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) @@ -183,13 +184,13 @@ for f in detect.get('files', {}).get('code', []): code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) if code_files: - result = extract(code_files, cache_root=Path(sys.argv[1])) + result = extract(code_files, cache_root=Path('INPUT_PATH')) Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') else: Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") print('No code files - skipping AST extraction') -" "INPUT_PATH" +" ``` #### Part B - Semantic extraction (parallel subagents) @@ -204,7 +205,7 @@ Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],' " ``` -**MANDATORY: You MUST use the subagent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the subagent tool you are doing this wrong.** +**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** Before dispatching subagents, print a timing estimate: - Load `total_words` and file counts from `graphify-out/.graphify_detect.json` @@ -220,7 +221,7 @@ SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from graphify.cache import check_semantic_cache from pathlib import Path @@ -230,7 +231,7 @@ detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encodin # every source file (#1392). Video is transcribed to a document in Step 2.5 first. all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root=sys.argv[1], prompt_file=sys.argv[2]) +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') # Always (re)write the cache file: write hits, else DELETE any leftover from a prior # run so Part C never merges a stale .graphify_cached.json (#1392). @@ -240,7 +241,7 @@ else: Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" "INPUT_PATH" "SPEC_PATH" +" ``` Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. @@ -282,7 +283,7 @@ Wait for all subagents. For each result: If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. -Merge all chunk files into `.graphify_semantic_new.json`. **After each subagent call completes, read the real token counts from the subagent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: +Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: ```bash $(cat graphify-out/.graphify_python) -c " import json, glob @@ -309,15 +310,15 @@ print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens' Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root=sys.argv[1], allowed_source_files=uncached, prompt_file=sys.argv[2]) +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') print(f'Cached {saved} files') -" "INPUT_PATH" "SPEC_PATH" +" ``` Merge cached + new results into `graphify-out/.graphify_semantic.json`: @@ -406,7 +407,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -431,7 +432,7 @@ if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') print('If this shrink is intentional (you deleted files), re-run a full build with --force.') raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, sys.argv[1], suggested_questions=questions) +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") analysis = { 'communities': {str(k): v for k, v in communities.items()}, @@ -442,7 +443,7 @@ analysis = { } Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" "INPUT_PATH" +" ``` If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. @@ -455,12 +456,12 @@ A non-destructive diagnostic on the extraction, before labeling. It surfaces edg ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from pathlib import Path from graphify.diagnostics import diagnose_extraction, format_diagnostic_report extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root=sys.argv[1]) +summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( ('dangling_endpoint_edges', 'dangling-endpoint edges'), @@ -470,7 +471,7 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0)] print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') -" "INPUT_PATH" +" ``` Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules). @@ -496,7 +497,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} @@ -507,7 +508,9 @@ labels = LABELS_DICT # Regenerate questions with real community labels (labels affect question phrasing) questions = suggest_questions(G, communities, labels) -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, sys.argv[1], suggested_questions=questions) +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") +Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") # Re-export so graph.json nodes carry the curated community_name (#2490). # Same extraction as Step 4, so the #479 shrink-guard passes on node count; # if it still refuses, surface the guard message - do not force past it. @@ -515,11 +518,8 @@ wrote = to_json(G, communities, 'graphify-out/graph.json', community_labels=labe if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') print('If this shrink is intentional (you deleted files), re-run a full build with --force.') - raise SystemExit(1) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") print('Report updated with community labels') -" "INPUT_PATH" +" ``` Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). @@ -534,14 +534,15 @@ If `--obsidian` was given: - If `--obsidian-dir ` was also given, pass it via `--dir`. Otherwise defaults to `graphify-out/obsidian`. ```bash -$(cat graphify-out/.graphify_python) -m graphify export obsidian -# Add `--dir ~/vaults/my-project` to use a custom directory. +graphify export obsidian +# or with custom dir: graphify export obsidian --dir ~/vaults/my-project ``` Generate the HTML graph (always, unless `--no-viz`): ```bash -$(cat graphify-out/.graphify_python) -m graphify export html # auto-aggregates to community view if graph > 5000 nodes +graphify export html # auto-aggregates to community view if graph > 5000 nodes +# or: graphify export html --no-viz ``` ### Steps 6b-8 - Wiki, Neo4j, FalkorDB, SVG, GraphML, MCP, benchmark (only on their flags) @@ -554,7 +555,7 @@ These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, ` ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from pathlib import Path from datetime import datetime, timezone from graphify.detect import save_manifest @@ -577,7 +578,7 @@ extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encod # types are gated on output. from graphify.cli import _stamped_manifest_files _corpus = detect.get('all_files') or detect['files'] -_manifest_files = _stamped_manifest_files(_corpus, extract, Path(sys.argv[1])) +_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) # Files dispatched this run (the changed subset) but NOT stamped above still carry # a stale semantic_hash from a prior run; clear it so detect_incremental re-queues # them instead of reading them as unchanged (#1948). @@ -589,7 +590,7 @@ _cleared = _dispatched - _stamped # files newly excluded since last run are dropped rather than masquerading as # deletions; untouched files' prior rows are still preserved (#1908). _scan = {f for fl in _corpus.values() for f in fl} -save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) # Update cumulative cost tracker input_tok = extract.get('input_tokens', 0) @@ -613,10 +614,10 @@ cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\" print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" "INPUT_PATH" +" rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null -rm -f graphify-out/needs_update 2>/dev/null || true +rm -f graphify-out/.needs_update 2>/dev/null || true ``` Replace INPUT_PATH with the actual path (same value used in Steps 4-5) so the manifest is relativized to the scan root. @@ -681,7 +682,7 @@ Both are non-default subcommands. `--update` re-extracts only new or changed fil When `graphify-out/graph.json` already exists and the user asks a question about the corpus, answer from the graph rather than rebuilding it: ```bash -$(cat graphify-out/.graphify_python) -m graphify query "" +graphify query "" ``` Before traversal, expand the question against the graph's own vocabulary so a wording mismatch does not collapse the answer to noise. If the `graphify query` CLI is unavailable, fall back to an inline NetworkX traversal of `graphify-out/graph.json`. Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. For that vocab-expansion step, the BFS/DFS traversal modes, the `--budget` cap, the NetworkX fallback, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`. diff --git a/graphify/skill-kilo.md b/graphify/skill-kilo.md index ef93d78a4d..3e70b050a4 100644 --- a/graphify/skill-kilo.md +++ b/graphify/skill-kilo.md @@ -97,7 +97,7 @@ fi mkdir -p graphify-out "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" # Save scan root so `graphify update` (no args) knows where to look next time -echo "$(cd "INPUT_PATH" && pwd)" > graphify-out/.graphify_root +echo "$(cd INPUT_PATH && pwd)" > graphify-out/.graphify_root ``` If the import succeeds, print nothing and move straight to Step 2. @@ -108,15 +108,15 @@ If the import succeeds, print nothing and move straight to Step 2. ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from graphify.detect import detect from pathlib import Path -result = detect(Path(sys.argv[1])) +result = detect(Path('INPUT_PATH')) # Write the sidecar from Python, not a shell redirect, so the same block renders # on PowerShell hosts without console-encoding drift (#2528). Path('graphify-out/.graphify_detect.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") print(f'Detected {result[\"total_files\"]} files') -" "INPUT_PATH" +" ``` Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: @@ -176,6 +176,7 @@ $(cat graphify-out/.graphify_python) -c " import sys, json from graphify.extract import collect_files, extract from pathlib import Path +import json code_files = [] detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) @@ -183,13 +184,13 @@ for f in detect.get('files', {}).get('code', []): code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) if code_files: - result = extract(code_files, cache_root=Path(sys.argv[1])) + result = extract(code_files, cache_root=Path('INPUT_PATH')) Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') else: Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") print('No code files - skipping AST extraction') -" "INPUT_PATH" +" ``` #### Part B - Semantic extraction (parallel subagents) @@ -204,7 +205,7 @@ Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],' " ``` -**MANDATORY: You MUST use the subagent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the subagent tool you are doing this wrong.** +**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** Before dispatching subagents, print a timing estimate: - Load `total_words` and file counts from `graphify-out/.graphify_detect.json` @@ -220,7 +221,7 @@ SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from graphify.cache import check_semantic_cache from pathlib import Path @@ -230,7 +231,7 @@ detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encodin # every source file (#1392). Video is transcribed to a document in Step 2.5 first. all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root=sys.argv[1], prompt_file=sys.argv[2]) +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') # Always (re)write the cache file: write hits, else DELETE any leftover from a prior # run so Part C never merges a stale .graphify_cached.json (#1392). @@ -240,7 +241,7 @@ else: Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" "INPUT_PATH" "SPEC_PATH" +" ``` Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. @@ -285,7 +286,7 @@ Wait for all subagents. For each result: If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. -Merge all chunk files into `.graphify_semantic_new.json`. **After each subagent call completes, read the real token counts from the subagent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: +Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: ```bash $(cat graphify-out/.graphify_python) -c " import json, glob @@ -312,15 +313,15 @@ print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens' Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root=sys.argv[1], allowed_source_files=uncached, prompt_file=sys.argv[2]) +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') print(f'Cached {saved} files') -" "INPUT_PATH" "SPEC_PATH" +" ``` Merge cached + new results into `graphify-out/.graphify_semantic.json`: @@ -409,7 +410,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -434,7 +435,7 @@ if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') print('If this shrink is intentional (you deleted files), re-run a full build with --force.') raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, sys.argv[1], suggested_questions=questions) +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") analysis = { 'communities': {str(k): v for k, v in communities.items()}, @@ -445,7 +446,7 @@ analysis = { } Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" "INPUT_PATH" +" ``` If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. @@ -458,12 +459,12 @@ A non-destructive diagnostic on the extraction, before labeling. It surfaces edg ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from pathlib import Path from graphify.diagnostics import diagnose_extraction, format_diagnostic_report extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root=sys.argv[1]) +summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( ('dangling_endpoint_edges', 'dangling-endpoint edges'), @@ -473,7 +474,7 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0)] print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') -" "INPUT_PATH" +" ``` Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules). @@ -499,7 +500,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} @@ -510,7 +511,9 @@ labels = LABELS_DICT # Regenerate questions with real community labels (labels affect question phrasing) questions = suggest_questions(G, communities, labels) -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, sys.argv[1], suggested_questions=questions) +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") +Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") # Re-export so graph.json nodes carry the curated community_name (#2490). # Same extraction as Step 4, so the #479 shrink-guard passes on node count; # if it still refuses, surface the guard message - do not force past it. @@ -518,11 +521,8 @@ wrote = to_json(G, communities, 'graphify-out/graph.json', community_labels=labe if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') print('If this shrink is intentional (you deleted files), re-run a full build with --force.') - raise SystemExit(1) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") print('Report updated with community labels') -" "INPUT_PATH" +" ``` Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). @@ -537,14 +537,15 @@ If `--obsidian` was given: - If `--obsidian-dir ` was also given, pass it via `--dir`. Otherwise defaults to `graphify-out/obsidian`. ```bash -$(cat graphify-out/.graphify_python) -m graphify export obsidian -# Add `--dir ~/vaults/my-project` to use a custom directory. +graphify export obsidian +# or with custom dir: graphify export obsidian --dir ~/vaults/my-project ``` Generate the HTML graph (always, unless `--no-viz`): ```bash -$(cat graphify-out/.graphify_python) -m graphify export html # auto-aggregates to community view if graph > 5000 nodes +graphify export html # auto-aggregates to community view if graph > 5000 nodes +# or: graphify export html --no-viz ``` ### Steps 6b-8 - Wiki, Neo4j, FalkorDB, SVG, GraphML, MCP, benchmark (only on their flags) @@ -557,7 +558,7 @@ These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, ` ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from pathlib import Path from datetime import datetime, timezone from graphify.detect import save_manifest @@ -580,7 +581,7 @@ extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encod # types are gated on output. from graphify.cli import _stamped_manifest_files _corpus = detect.get('all_files') or detect['files'] -_manifest_files = _stamped_manifest_files(_corpus, extract, Path(sys.argv[1])) +_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) # Files dispatched this run (the changed subset) but NOT stamped above still carry # a stale semantic_hash from a prior run; clear it so detect_incremental re-queues # them instead of reading them as unchanged (#1948). @@ -592,7 +593,7 @@ _cleared = _dispatched - _stamped # files newly excluded since last run are dropped rather than masquerading as # deletions; untouched files' prior rows are still preserved (#1908). _scan = {f for fl in _corpus.values() for f in fl} -save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) # Update cumulative cost tracker input_tok = extract.get('input_tokens', 0) @@ -616,10 +617,10 @@ cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\" print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" "INPUT_PATH" +" rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null -rm -f graphify-out/needs_update 2>/dev/null || true +rm -f graphify-out/.needs_update 2>/dev/null || true ``` Replace INPUT_PATH with the actual path (same value used in Steps 4-5) so the manifest is relativized to the scan root. @@ -684,7 +685,7 @@ Both are non-default subcommands. `--update` re-extracts only new or changed fil When `graphify-out/graph.json` already exists and the user asks a question about the corpus, answer from the graph rather than rebuilding it: ```bash -$(cat graphify-out/.graphify_python) -m graphify query "" +graphify query "" ``` Before traversal, expand the question against the graph's own vocabulary so a wording mismatch does not collapse the answer to noise. If the `graphify query` CLI is unavailable, fall back to an inline NetworkX traversal of `graphify-out/graph.json`. Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. For that vocab-expansion step, the BFS/DFS traversal modes, the `--budget` cap, the NetworkX fallback, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`. diff --git a/graphify/skill-kiro.md b/graphify/skill-kiro.md index 55b77d6828..abd2811d23 100644 --- a/graphify/skill-kiro.md +++ b/graphify/skill-kiro.md @@ -97,7 +97,7 @@ fi mkdir -p graphify-out "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" # Save scan root so `graphify update` (no args) knows where to look next time -echo "$(cd "INPUT_PATH" && pwd)" > graphify-out/.graphify_root +echo "$(cd INPUT_PATH && pwd)" > graphify-out/.graphify_root ``` If the import succeeds, print nothing and move straight to Step 2. @@ -108,15 +108,15 @@ If the import succeeds, print nothing and move straight to Step 2. ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from graphify.detect import detect from pathlib import Path -result = detect(Path(sys.argv[1])) +result = detect(Path('INPUT_PATH')) # Write the sidecar from Python, not a shell redirect, so the same block renders # on PowerShell hosts without console-encoding drift (#2528). Path('graphify-out/.graphify_detect.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") print(f'Detected {result[\"total_files\"]} files') -" "INPUT_PATH" +" ``` Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: @@ -176,6 +176,7 @@ $(cat graphify-out/.graphify_python) -c " import sys, json from graphify.extract import collect_files, extract from pathlib import Path +import json code_files = [] detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) @@ -183,13 +184,13 @@ for f in detect.get('files', {}).get('code', []): code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) if code_files: - result = extract(code_files, cache_root=Path(sys.argv[1])) + result = extract(code_files, cache_root=Path('INPUT_PATH')) Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') else: Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") print('No code files - skipping AST extraction') -" "INPUT_PATH" +" ``` #### Part B - Semantic extraction (parallel subagents) @@ -204,7 +205,7 @@ Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],' " ``` -**MANDATORY: You MUST use the subagent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the subagent tool you are doing this wrong.** +**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** Before dispatching subagents, print a timing estimate: - Load `total_words` and file counts from `graphify-out/.graphify_detect.json` @@ -220,7 +221,7 @@ SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from graphify.cache import check_semantic_cache from pathlib import Path @@ -230,7 +231,7 @@ detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encodin # every source file (#1392). Video is transcribed to a document in Step 2.5 first. all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root=sys.argv[1], prompt_file=sys.argv[2]) +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') # Always (re)write the cache file: write hits, else DELETE any leftover from a prior # run so Part C never merges a stale .graphify_cached.json (#1392). @@ -240,7 +241,7 @@ else: Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" "INPUT_PATH" "SPEC_PATH" +" ``` Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. @@ -285,7 +286,7 @@ Wait for all subagents. For each result: If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. -Merge all chunk files into `.graphify_semantic_new.json`. **After each subagent call completes, read the real token counts from the subagent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: +Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: ```bash $(cat graphify-out/.graphify_python) -c " import json, glob @@ -312,15 +313,15 @@ print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens' Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root=sys.argv[1], allowed_source_files=uncached, prompt_file=sys.argv[2]) +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') print(f'Cached {saved} files') -" "INPUT_PATH" "SPEC_PATH" +" ``` Merge cached + new results into `graphify-out/.graphify_semantic.json`: @@ -409,7 +410,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -434,7 +435,7 @@ if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') print('If this shrink is intentional (you deleted files), re-run a full build with --force.') raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, sys.argv[1], suggested_questions=questions) +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") analysis = { 'communities': {str(k): v for k, v in communities.items()}, @@ -445,7 +446,7 @@ analysis = { } Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" "INPUT_PATH" +" ``` If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. @@ -458,12 +459,12 @@ A non-destructive diagnostic on the extraction, before labeling. It surfaces edg ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from pathlib import Path from graphify.diagnostics import diagnose_extraction, format_diagnostic_report extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root=sys.argv[1]) +summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( ('dangling_endpoint_edges', 'dangling-endpoint edges'), @@ -473,7 +474,7 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0)] print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') -" "INPUT_PATH" +" ``` Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules). @@ -499,7 +500,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} @@ -510,7 +511,9 @@ labels = LABELS_DICT # Regenerate questions with real community labels (labels affect question phrasing) questions = suggest_questions(G, communities, labels) -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, sys.argv[1], suggested_questions=questions) +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") +Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") # Re-export so graph.json nodes carry the curated community_name (#2490). # Same extraction as Step 4, so the #479 shrink-guard passes on node count; # if it still refuses, surface the guard message - do not force past it. @@ -518,11 +521,8 @@ wrote = to_json(G, communities, 'graphify-out/graph.json', community_labels=labe if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') print('If this shrink is intentional (you deleted files), re-run a full build with --force.') - raise SystemExit(1) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") print('Report updated with community labels') -" "INPUT_PATH" +" ``` Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). @@ -537,14 +537,15 @@ If `--obsidian` was given: - If `--obsidian-dir ` was also given, pass it via `--dir`. Otherwise defaults to `graphify-out/obsidian`. ```bash -$(cat graphify-out/.graphify_python) -m graphify export obsidian -# Add `--dir ~/vaults/my-project` to use a custom directory. +graphify export obsidian +# or with custom dir: graphify export obsidian --dir ~/vaults/my-project ``` Generate the HTML graph (always, unless `--no-viz`): ```bash -$(cat graphify-out/.graphify_python) -m graphify export html # auto-aggregates to community view if graph > 5000 nodes +graphify export html # auto-aggregates to community view if graph > 5000 nodes +# or: graphify export html --no-viz ``` ### Steps 6b-8 - Wiki, Neo4j, FalkorDB, SVG, GraphML, MCP, benchmark (only on their flags) @@ -557,7 +558,7 @@ These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, ` ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from pathlib import Path from datetime import datetime, timezone from graphify.detect import save_manifest @@ -580,7 +581,7 @@ extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encod # types are gated on output. from graphify.cli import _stamped_manifest_files _corpus = detect.get('all_files') or detect['files'] -_manifest_files = _stamped_manifest_files(_corpus, extract, Path(sys.argv[1])) +_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) # Files dispatched this run (the changed subset) but NOT stamped above still carry # a stale semantic_hash from a prior run; clear it so detect_incremental re-queues # them instead of reading them as unchanged (#1948). @@ -592,7 +593,7 @@ _cleared = _dispatched - _stamped # files newly excluded since last run are dropped rather than masquerading as # deletions; untouched files' prior rows are still preserved (#1908). _scan = {f for fl in _corpus.values() for f in fl} -save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) # Update cumulative cost tracker input_tok = extract.get('input_tokens', 0) @@ -616,10 +617,10 @@ cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\" print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" "INPUT_PATH" +" rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null -rm -f graphify-out/needs_update 2>/dev/null || true +rm -f graphify-out/.needs_update 2>/dev/null || true ``` Replace INPUT_PATH with the actual path (same value used in Steps 4-5) so the manifest is relativized to the scan root. @@ -684,7 +685,7 @@ Both are non-default subcommands. `--update` re-extracts only new or changed fil When `graphify-out/graph.json` already exists and the user asks a question about the corpus, answer from the graph rather than rebuilding it: ```bash -$(cat graphify-out/.graphify_python) -m graphify query "" +graphify query "" ``` Before traversal, expand the question against the graph's own vocabulary so a wording mismatch does not collapse the answer to noise. If the `graphify query` CLI is unavailable, fall back to an inline NetworkX traversal of `graphify-out/graph.json`. Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. For that vocab-expansion step, the BFS/DFS traversal modes, the `--budget` cap, the NetworkX fallback, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`. diff --git a/graphify/skill-opencode.md b/graphify/skill-opencode.md index f1afcb1aa3..91ced60675 100644 --- a/graphify/skill-opencode.md +++ b/graphify/skill-opencode.md @@ -97,7 +97,7 @@ fi mkdir -p graphify-out "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" # Save scan root so `graphify update` (no args) knows where to look next time -echo "$(cd "INPUT_PATH" && pwd)" > graphify-out/.graphify_root +echo "$(cd INPUT_PATH && pwd)" > graphify-out/.graphify_root ``` If the import succeeds, print nothing and move straight to Step 2. @@ -108,15 +108,15 @@ If the import succeeds, print nothing and move straight to Step 2. ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from graphify.detect import detect from pathlib import Path -result = detect(Path(sys.argv[1])) +result = detect(Path('INPUT_PATH')) # Write the sidecar from Python, not a shell redirect, so the same block renders # on PowerShell hosts without console-encoding drift (#2528). Path('graphify-out/.graphify_detect.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") print(f'Detected {result[\"total_files\"]} files') -" "INPUT_PATH" +" ``` Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: @@ -176,6 +176,7 @@ $(cat graphify-out/.graphify_python) -c " import sys, json from graphify.extract import collect_files, extract from pathlib import Path +import json code_files = [] detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) @@ -183,13 +184,13 @@ for f in detect.get('files', {}).get('code', []): code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) if code_files: - result = extract(code_files, cache_root=Path(sys.argv[1])) + result = extract(code_files, cache_root=Path('INPUT_PATH')) Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') else: Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") print('No code files - skipping AST extraction') -" "INPUT_PATH" +" ``` #### Part B - Semantic extraction (parallel subagents) @@ -204,7 +205,7 @@ Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],' " ``` -**MANDATORY: You MUST use the subagent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the subagent tool you are doing this wrong.** +**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** Before dispatching subagents, print a timing estimate: - Load `total_words` and file counts from `graphify-out/.graphify_detect.json` @@ -220,7 +221,7 @@ SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from graphify.cache import check_semantic_cache from pathlib import Path @@ -230,7 +231,7 @@ detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encodin # every source file (#1392). Video is transcribed to a document in Step 2.5 first. all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root=sys.argv[1], prompt_file=sys.argv[2]) +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') # Always (re)write the cache file: write hits, else DELETE any leftover from a prior # run so Part C never merges a stale .graphify_cached.json (#1392). @@ -240,7 +241,7 @@ else: Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" "INPUT_PATH" "SPEC_PATH" +" ``` Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. @@ -277,7 +278,7 @@ Wait for all subagents. For each result: If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. -Merge all chunk files into `.graphify_semantic_new.json`. **After each subagent call completes, read the real token counts from the subagent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: +Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: ```bash $(cat graphify-out/.graphify_python) -c " import json, glob @@ -304,15 +305,15 @@ print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens' Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root=sys.argv[1], allowed_source_files=uncached, prompt_file=sys.argv[2]) +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') print(f'Cached {saved} files') -" "INPUT_PATH" "SPEC_PATH" +" ``` Merge cached + new results into `graphify-out/.graphify_semantic.json`: @@ -401,7 +402,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -426,7 +427,7 @@ if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') print('If this shrink is intentional (you deleted files), re-run a full build with --force.') raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, sys.argv[1], suggested_questions=questions) +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") analysis = { 'communities': {str(k): v for k, v in communities.items()}, @@ -437,7 +438,7 @@ analysis = { } Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" "INPUT_PATH" +" ``` If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. @@ -450,12 +451,12 @@ A non-destructive diagnostic on the extraction, before labeling. It surfaces edg ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from pathlib import Path from graphify.diagnostics import diagnose_extraction, format_diagnostic_report extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root=sys.argv[1]) +summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( ('dangling_endpoint_edges', 'dangling-endpoint edges'), @@ -465,7 +466,7 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0)] print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') -" "INPUT_PATH" +" ``` Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules). @@ -491,7 +492,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} @@ -502,7 +503,9 @@ labels = LABELS_DICT # Regenerate questions with real community labels (labels affect question phrasing) questions = suggest_questions(G, communities, labels) -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, sys.argv[1], suggested_questions=questions) +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") +Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") # Re-export so graph.json nodes carry the curated community_name (#2490). # Same extraction as Step 4, so the #479 shrink-guard passes on node count; # if it still refuses, surface the guard message - do not force past it. @@ -510,11 +513,8 @@ wrote = to_json(G, communities, 'graphify-out/graph.json', community_labels=labe if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') print('If this shrink is intentional (you deleted files), re-run a full build with --force.') - raise SystemExit(1) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") print('Report updated with community labels') -" "INPUT_PATH" +" ``` Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). @@ -529,14 +529,15 @@ If `--obsidian` was given: - If `--obsidian-dir ` was also given, pass it via `--dir`. Otherwise defaults to `graphify-out/obsidian`. ```bash -$(cat graphify-out/.graphify_python) -m graphify export obsidian -# Add `--dir ~/vaults/my-project` to use a custom directory. +graphify export obsidian +# or with custom dir: graphify export obsidian --dir ~/vaults/my-project ``` Generate the HTML graph (always, unless `--no-viz`): ```bash -$(cat graphify-out/.graphify_python) -m graphify export html # auto-aggregates to community view if graph > 5000 nodes +graphify export html # auto-aggregates to community view if graph > 5000 nodes +# or: graphify export html --no-viz ``` ### Steps 6b-8 - Wiki, Neo4j, FalkorDB, SVG, GraphML, MCP, benchmark (only on their flags) @@ -549,7 +550,7 @@ These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, ` ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from pathlib import Path from datetime import datetime, timezone from graphify.detect import save_manifest @@ -572,7 +573,7 @@ extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encod # types are gated on output. from graphify.cli import _stamped_manifest_files _corpus = detect.get('all_files') or detect['files'] -_manifest_files = _stamped_manifest_files(_corpus, extract, Path(sys.argv[1])) +_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) # Files dispatched this run (the changed subset) but NOT stamped above still carry # a stale semantic_hash from a prior run; clear it so detect_incremental re-queues # them instead of reading them as unchanged (#1948). @@ -584,7 +585,7 @@ _cleared = _dispatched - _stamped # files newly excluded since last run are dropped rather than masquerading as # deletions; untouched files' prior rows are still preserved (#1908). _scan = {f for fl in _corpus.values() for f in fl} -save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) # Update cumulative cost tracker input_tok = extract.get('input_tokens', 0) @@ -608,10 +609,10 @@ cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\" print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" "INPUT_PATH" +" rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null -rm -f graphify-out/needs_update 2>/dev/null || true +rm -f graphify-out/.needs_update 2>/dev/null || true ``` Replace INPUT_PATH with the actual path (same value used in Steps 4-5) so the manifest is relativized to the scan root. @@ -676,7 +677,7 @@ Both are non-default subcommands. `--update` re-extracts only new or changed fil When `graphify-out/graph.json` already exists and the user asks a question about the corpus, answer from the graph rather than rebuilding it: ```bash -$(cat graphify-out/.graphify_python) -m graphify query "" +graphify query "" ``` Before traversal, expand the question against the graph's own vocabulary so a wording mismatch does not collapse the answer to noise. If the `graphify query` CLI is unavailable, fall back to an inline NetworkX traversal of `graphify-out/graph.json`. Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. For that vocab-expansion step, the BFS/DFS traversal modes, the `--budget` cap, the NetworkX fallback, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`. diff --git a/graphify/skill-pi.md b/graphify/skill-pi.md index 55b77d6828..abd2811d23 100644 --- a/graphify/skill-pi.md +++ b/graphify/skill-pi.md @@ -97,7 +97,7 @@ fi mkdir -p graphify-out "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" # Save scan root so `graphify update` (no args) knows where to look next time -echo "$(cd "INPUT_PATH" && pwd)" > graphify-out/.graphify_root +echo "$(cd INPUT_PATH && pwd)" > graphify-out/.graphify_root ``` If the import succeeds, print nothing and move straight to Step 2. @@ -108,15 +108,15 @@ If the import succeeds, print nothing and move straight to Step 2. ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from graphify.detect import detect from pathlib import Path -result = detect(Path(sys.argv[1])) +result = detect(Path('INPUT_PATH')) # Write the sidecar from Python, not a shell redirect, so the same block renders # on PowerShell hosts without console-encoding drift (#2528). Path('graphify-out/.graphify_detect.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") print(f'Detected {result[\"total_files\"]} files') -" "INPUT_PATH" +" ``` Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: @@ -176,6 +176,7 @@ $(cat graphify-out/.graphify_python) -c " import sys, json from graphify.extract import collect_files, extract from pathlib import Path +import json code_files = [] detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) @@ -183,13 +184,13 @@ for f in detect.get('files', {}).get('code', []): code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) if code_files: - result = extract(code_files, cache_root=Path(sys.argv[1])) + result = extract(code_files, cache_root=Path('INPUT_PATH')) Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') else: Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") print('No code files - skipping AST extraction') -" "INPUT_PATH" +" ``` #### Part B - Semantic extraction (parallel subagents) @@ -204,7 +205,7 @@ Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],' " ``` -**MANDATORY: You MUST use the subagent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the subagent tool you are doing this wrong.** +**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** Before dispatching subagents, print a timing estimate: - Load `total_words` and file counts from `graphify-out/.graphify_detect.json` @@ -220,7 +221,7 @@ SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from graphify.cache import check_semantic_cache from pathlib import Path @@ -230,7 +231,7 @@ detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encodin # every source file (#1392). Video is transcribed to a document in Step 2.5 first. all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root=sys.argv[1], prompt_file=sys.argv[2]) +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') # Always (re)write the cache file: write hits, else DELETE any leftover from a prior # run so Part C never merges a stale .graphify_cached.json (#1392). @@ -240,7 +241,7 @@ else: Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" "INPUT_PATH" "SPEC_PATH" +" ``` Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. @@ -285,7 +286,7 @@ Wait for all subagents. For each result: If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. -Merge all chunk files into `.graphify_semantic_new.json`. **After each subagent call completes, read the real token counts from the subagent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: +Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: ```bash $(cat graphify-out/.graphify_python) -c " import json, glob @@ -312,15 +313,15 @@ print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens' Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root=sys.argv[1], allowed_source_files=uncached, prompt_file=sys.argv[2]) +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') print(f'Cached {saved} files') -" "INPUT_PATH" "SPEC_PATH" +" ``` Merge cached + new results into `graphify-out/.graphify_semantic.json`: @@ -409,7 +410,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -434,7 +435,7 @@ if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') print('If this shrink is intentional (you deleted files), re-run a full build with --force.') raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, sys.argv[1], suggested_questions=questions) +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") analysis = { 'communities': {str(k): v for k, v in communities.items()}, @@ -445,7 +446,7 @@ analysis = { } Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" "INPUT_PATH" +" ``` If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. @@ -458,12 +459,12 @@ A non-destructive diagnostic on the extraction, before labeling. It surfaces edg ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from pathlib import Path from graphify.diagnostics import diagnose_extraction, format_diagnostic_report extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root=sys.argv[1]) +summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( ('dangling_endpoint_edges', 'dangling-endpoint edges'), @@ -473,7 +474,7 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0)] print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') -" "INPUT_PATH" +" ``` Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules). @@ -499,7 +500,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} @@ -510,7 +511,9 @@ labels = LABELS_DICT # Regenerate questions with real community labels (labels affect question phrasing) questions = suggest_questions(G, communities, labels) -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, sys.argv[1], suggested_questions=questions) +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") +Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") # Re-export so graph.json nodes carry the curated community_name (#2490). # Same extraction as Step 4, so the #479 shrink-guard passes on node count; # if it still refuses, surface the guard message - do not force past it. @@ -518,11 +521,8 @@ wrote = to_json(G, communities, 'graphify-out/graph.json', community_labels=labe if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') print('If this shrink is intentional (you deleted files), re-run a full build with --force.') - raise SystemExit(1) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") print('Report updated with community labels') -" "INPUT_PATH" +" ``` Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). @@ -537,14 +537,15 @@ If `--obsidian` was given: - If `--obsidian-dir ` was also given, pass it via `--dir`. Otherwise defaults to `graphify-out/obsidian`. ```bash -$(cat graphify-out/.graphify_python) -m graphify export obsidian -# Add `--dir ~/vaults/my-project` to use a custom directory. +graphify export obsidian +# or with custom dir: graphify export obsidian --dir ~/vaults/my-project ``` Generate the HTML graph (always, unless `--no-viz`): ```bash -$(cat graphify-out/.graphify_python) -m graphify export html # auto-aggregates to community view if graph > 5000 nodes +graphify export html # auto-aggregates to community view if graph > 5000 nodes +# or: graphify export html --no-viz ``` ### Steps 6b-8 - Wiki, Neo4j, FalkorDB, SVG, GraphML, MCP, benchmark (only on their flags) @@ -557,7 +558,7 @@ These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, ` ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from pathlib import Path from datetime import datetime, timezone from graphify.detect import save_manifest @@ -580,7 +581,7 @@ extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encod # types are gated on output. from graphify.cli import _stamped_manifest_files _corpus = detect.get('all_files') or detect['files'] -_manifest_files = _stamped_manifest_files(_corpus, extract, Path(sys.argv[1])) +_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) # Files dispatched this run (the changed subset) but NOT stamped above still carry # a stale semantic_hash from a prior run; clear it so detect_incremental re-queues # them instead of reading them as unchanged (#1948). @@ -592,7 +593,7 @@ _cleared = _dispatched - _stamped # files newly excluded since last run are dropped rather than masquerading as # deletions; untouched files' prior rows are still preserved (#1908). _scan = {f for fl in _corpus.values() for f in fl} -save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) # Update cumulative cost tracker input_tok = extract.get('input_tokens', 0) @@ -616,10 +617,10 @@ cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\" print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" "INPUT_PATH" +" rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null -rm -f graphify-out/needs_update 2>/dev/null || true +rm -f graphify-out/.needs_update 2>/dev/null || true ``` Replace INPUT_PATH with the actual path (same value used in Steps 4-5) so the manifest is relativized to the scan root. @@ -684,7 +685,7 @@ Both are non-default subcommands. `--update` re-extracts only new or changed fil When `graphify-out/graph.json` already exists and the user asks a question about the corpus, answer from the graph rather than rebuilding it: ```bash -$(cat graphify-out/.graphify_python) -m graphify query "" +graphify query "" ``` Before traversal, expand the question against the graph's own vocabulary so a wording mismatch does not collapse the answer to noise. If the `graphify query` CLI is unavailable, fall back to an inline NetworkX traversal of `graphify-out/graph.json`. Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. For that vocab-expansion step, the BFS/DFS traversal modes, the `--budget` cap, the NetworkX fallback, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`. diff --git a/graphify/skill-trae.md b/graphify/skill-trae.md index 052dc548af..050667bc20 100644 --- a/graphify/skill-trae.md +++ b/graphify/skill-trae.md @@ -97,7 +97,7 @@ fi mkdir -p graphify-out "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" # Save scan root so `graphify update` (no args) knows where to look next time -echo "$(cd "INPUT_PATH" && pwd)" > graphify-out/.graphify_root +echo "$(cd INPUT_PATH && pwd)" > graphify-out/.graphify_root ``` If the import succeeds, print nothing and move straight to Step 2. @@ -108,15 +108,15 @@ If the import succeeds, print nothing and move straight to Step 2. ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from graphify.detect import detect from pathlib import Path -result = detect(Path(sys.argv[1])) +result = detect(Path('INPUT_PATH')) # Write the sidecar from Python, not a shell redirect, so the same block renders # on PowerShell hosts without console-encoding drift (#2528). Path('graphify-out/.graphify_detect.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") print(f'Detected {result[\"total_files\"]} files') -" "INPUT_PATH" +" ``` Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: @@ -176,6 +176,7 @@ $(cat graphify-out/.graphify_python) -c " import sys, json from graphify.extract import collect_files, extract from pathlib import Path +import json code_files = [] detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) @@ -183,13 +184,13 @@ for f in detect.get('files', {}).get('code', []): code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) if code_files: - result = extract(code_files, cache_root=Path(sys.argv[1])) + result = extract(code_files, cache_root=Path('INPUT_PATH')) Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') else: Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") print('No code files - skipping AST extraction') -" "INPUT_PATH" +" ``` #### Part B - Semantic extraction (parallel subagents) @@ -204,7 +205,7 @@ Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],' " ``` -**MANDATORY: You MUST use the subagent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the subagent tool you are doing this wrong.** +**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** Before dispatching subagents, print a timing estimate: - Load `total_words` and file counts from `graphify-out/.graphify_detect.json` @@ -220,7 +221,7 @@ SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from graphify.cache import check_semantic_cache from pathlib import Path @@ -230,7 +231,7 @@ detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encodin # every source file (#1392). Video is transcribed to a document in Step 2.5 first. all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root=sys.argv[1], prompt_file=sys.argv[2]) +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') # Always (re)write the cache file: write hits, else DELETE any leftover from a prior # run so Part C never merges a stale .graphify_cached.json (#1392). @@ -240,7 +241,7 @@ else: Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" "INPUT_PATH" "SPEC_PATH" +" ``` Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. @@ -283,7 +284,7 @@ Wait for all subagents. For each result: If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. -Merge all chunk files into `.graphify_semantic_new.json`. **After each subagent call completes, read the real token counts from the subagent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: +Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: ```bash $(cat graphify-out/.graphify_python) -c " import json, glob @@ -310,15 +311,15 @@ print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens' Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root=sys.argv[1], allowed_source_files=uncached, prompt_file=sys.argv[2]) +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') print(f'Cached {saved} files') -" "INPUT_PATH" "SPEC_PATH" +" ``` Merge cached + new results into `graphify-out/.graphify_semantic.json`: @@ -407,7 +408,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -432,7 +433,7 @@ if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') print('If this shrink is intentional (you deleted files), re-run a full build with --force.') raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, sys.argv[1], suggested_questions=questions) +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") analysis = { 'communities': {str(k): v for k, v in communities.items()}, @@ -443,7 +444,7 @@ analysis = { } Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" "INPUT_PATH" +" ``` If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. @@ -456,12 +457,12 @@ A non-destructive diagnostic on the extraction, before labeling. It surfaces edg ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from pathlib import Path from graphify.diagnostics import diagnose_extraction, format_diagnostic_report extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root=sys.argv[1]) +summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( ('dangling_endpoint_edges', 'dangling-endpoint edges'), @@ -471,7 +472,7 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0)] print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') -" "INPUT_PATH" +" ``` Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules). @@ -497,7 +498,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} @@ -508,7 +509,9 @@ labels = LABELS_DICT # Regenerate questions with real community labels (labels affect question phrasing) questions = suggest_questions(G, communities, labels) -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, sys.argv[1], suggested_questions=questions) +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") +Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") # Re-export so graph.json nodes carry the curated community_name (#2490). # Same extraction as Step 4, so the #479 shrink-guard passes on node count; # if it still refuses, surface the guard message - do not force past it. @@ -516,11 +519,8 @@ wrote = to_json(G, communities, 'graphify-out/graph.json', community_labels=labe if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') print('If this shrink is intentional (you deleted files), re-run a full build with --force.') - raise SystemExit(1) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") print('Report updated with community labels') -" "INPUT_PATH" +" ``` Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). @@ -535,14 +535,15 @@ If `--obsidian` was given: - If `--obsidian-dir ` was also given, pass it via `--dir`. Otherwise defaults to `graphify-out/obsidian`. ```bash -$(cat graphify-out/.graphify_python) -m graphify export obsidian -# Add `--dir ~/vaults/my-project` to use a custom directory. +graphify export obsidian +# or with custom dir: graphify export obsidian --dir ~/vaults/my-project ``` Generate the HTML graph (always, unless `--no-viz`): ```bash -$(cat graphify-out/.graphify_python) -m graphify export html # auto-aggregates to community view if graph > 5000 nodes +graphify export html # auto-aggregates to community view if graph > 5000 nodes +# or: graphify export html --no-viz ``` ### Steps 6b-8 - Wiki, Neo4j, FalkorDB, SVG, GraphML, MCP, benchmark (only on their flags) @@ -555,7 +556,7 @@ These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, ` ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from pathlib import Path from datetime import datetime, timezone from graphify.detect import save_manifest @@ -578,7 +579,7 @@ extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encod # types are gated on output. from graphify.cli import _stamped_manifest_files _corpus = detect.get('all_files') or detect['files'] -_manifest_files = _stamped_manifest_files(_corpus, extract, Path(sys.argv[1])) +_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) # Files dispatched this run (the changed subset) but NOT stamped above still carry # a stale semantic_hash from a prior run; clear it so detect_incremental re-queues # them instead of reading them as unchanged (#1948). @@ -590,7 +591,7 @@ _cleared = _dispatched - _stamped # files newly excluded since last run are dropped rather than masquerading as # deletions; untouched files' prior rows are still preserved (#1908). _scan = {f for fl in _corpus.values() for f in fl} -save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) # Update cumulative cost tracker input_tok = extract.get('input_tokens', 0) @@ -614,10 +615,10 @@ cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\" print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" "INPUT_PATH" +" rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null -rm -f graphify-out/needs_update 2>/dev/null || true +rm -f graphify-out/.needs_update 2>/dev/null || true ``` Replace INPUT_PATH with the actual path (same value used in Steps 4-5) so the manifest is relativized to the scan root. @@ -682,7 +683,7 @@ Both are non-default subcommands. `--update` re-extracts only new or changed fil When `graphify-out/graph.json` already exists and the user asks a question about the corpus, answer from the graph rather than rebuilding it: ```bash -$(cat graphify-out/.graphify_python) -m graphify query "" +graphify query "" ``` Before traversal, expand the question against the graph's own vocabulary so a wording mismatch does not collapse the answer to noise. If the `graphify query` CLI is unavailable, fall back to an inline NetworkX traversal of `graphify-out/graph.json`. Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. For that vocab-expansion step, the BFS/DFS traversal modes, the `--budget` cap, the NetworkX fallback, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`. diff --git a/graphify/skill-vscode.md b/graphify/skill-vscode.md index a251e12c02..20c7c0835c 100644 --- a/graphify/skill-vscode.md +++ b/graphify/skill-vscode.md @@ -97,7 +97,7 @@ fi mkdir -p graphify-out "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" # Save scan root so `graphify update` (no args) knows where to look next time -echo "$(cd "INPUT_PATH" && pwd)" > graphify-out/.graphify_root +echo "$(cd INPUT_PATH && pwd)" > graphify-out/.graphify_root ``` If the import succeeds, print nothing and move straight to Step 2. @@ -108,15 +108,15 @@ If the import succeeds, print nothing and move straight to Step 2. ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from graphify.detect import detect from pathlib import Path -result = detect(Path(sys.argv[1])) +result = detect(Path('INPUT_PATH')) # Write the sidecar from Python, not a shell redirect, so the same block renders # on PowerShell hosts without console-encoding drift (#2528). Path('graphify-out/.graphify_detect.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") print(f'Detected {result[\"total_files\"]} files') -" "INPUT_PATH" +" ``` Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: @@ -176,6 +176,7 @@ $(cat graphify-out/.graphify_python) -c " import sys, json from graphify.extract import collect_files, extract from pathlib import Path +import json code_files = [] detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) @@ -183,13 +184,13 @@ for f in detect.get('files', {}).get('code', []): code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) if code_files: - result = extract(code_files, cache_root=Path(sys.argv[1])) + result = extract(code_files, cache_root=Path('INPUT_PATH')) Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') else: Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") print('No code files - skipping AST extraction') -" "INPUT_PATH" +" ``` #### Part B - Semantic extraction (parallel subagents) @@ -204,7 +205,7 @@ Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],' " ``` -**MANDATORY: You MUST use the subagent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the subagent tool you are doing this wrong.** +**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** Before dispatching subagents, print a timing estimate: - Load `total_words` and file counts from `graphify-out/.graphify_detect.json` @@ -220,7 +221,7 @@ SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from graphify.cache import check_semantic_cache from pathlib import Path @@ -230,7 +231,7 @@ detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encodin # every source file (#1392). Video is transcribed to a document in Step 2.5 first. all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root=sys.argv[1], prompt_file=sys.argv[2]) +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') # Always (re)write the cache file: write hits, else DELETE any leftover from a prior # run so Part C never merges a stale .graphify_cached.json (#1392). @@ -240,7 +241,7 @@ else: Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" "INPUT_PATH" "SPEC_PATH" +" ``` Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. @@ -281,7 +282,7 @@ Wait for all subagents. For each result: If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. -Merge all chunk files into `.graphify_semantic_new.json`. **After each subagent call completes, read the real token counts from the subagent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: +Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: ```bash $(cat graphify-out/.graphify_python) -c " import json, glob @@ -308,15 +309,15 @@ print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens' Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root=sys.argv[1], allowed_source_files=uncached, prompt_file=sys.argv[2]) +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') print(f'Cached {saved} files') -" "INPUT_PATH" "SPEC_PATH" +" ``` Merge cached + new results into `graphify-out/.graphify_semantic.json`: @@ -405,7 +406,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -430,7 +431,7 @@ if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') print('If this shrink is intentional (you deleted files), re-run a full build with --force.') raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, sys.argv[1], suggested_questions=questions) +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") analysis = { 'communities': {str(k): v for k, v in communities.items()}, @@ -441,7 +442,7 @@ analysis = { } Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" "INPUT_PATH" +" ``` If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. @@ -454,12 +455,12 @@ A non-destructive diagnostic on the extraction, before labeling. It surfaces edg ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from pathlib import Path from graphify.diagnostics import diagnose_extraction, format_diagnostic_report extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root=sys.argv[1]) +summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( ('dangling_endpoint_edges', 'dangling-endpoint edges'), @@ -469,7 +470,7 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0)] print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') -" "INPUT_PATH" +" ``` Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules). @@ -495,7 +496,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} @@ -506,7 +507,9 @@ labels = LABELS_DICT # Regenerate questions with real community labels (labels affect question phrasing) questions = suggest_questions(G, communities, labels) -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, sys.argv[1], suggested_questions=questions) +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") +Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") # Re-export so graph.json nodes carry the curated community_name (#2490). # Same extraction as Step 4, so the #479 shrink-guard passes on node count; # if it still refuses, surface the guard message - do not force past it. @@ -514,11 +517,8 @@ wrote = to_json(G, communities, 'graphify-out/graph.json', community_labels=labe if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') print('If this shrink is intentional (you deleted files), re-run a full build with --force.') - raise SystemExit(1) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") print('Report updated with community labels') -" "INPUT_PATH" +" ``` Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). @@ -533,14 +533,15 @@ If `--obsidian` was given: - If `--obsidian-dir ` was also given, pass it via `--dir`. Otherwise defaults to `graphify-out/obsidian`. ```bash -$(cat graphify-out/.graphify_python) -m graphify export obsidian -# Add `--dir ~/vaults/my-project` to use a custom directory. +graphify export obsidian +# or with custom dir: graphify export obsidian --dir ~/vaults/my-project ``` Generate the HTML graph (always, unless `--no-viz`): ```bash -$(cat graphify-out/.graphify_python) -m graphify export html # auto-aggregates to community view if graph > 5000 nodes +graphify export html # auto-aggregates to community view if graph > 5000 nodes +# or: graphify export html --no-viz ``` ### Steps 6b-8 - Wiki, Neo4j, FalkorDB, SVG, GraphML, MCP, benchmark (only on their flags) @@ -553,7 +554,7 @@ These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, ` ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from pathlib import Path from datetime import datetime, timezone from graphify.detect import save_manifest @@ -576,7 +577,7 @@ extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encod # types are gated on output. from graphify.cli import _stamped_manifest_files _corpus = detect.get('all_files') or detect['files'] -_manifest_files = _stamped_manifest_files(_corpus, extract, Path(sys.argv[1])) +_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) # Files dispatched this run (the changed subset) but NOT stamped above still carry # a stale semantic_hash from a prior run; clear it so detect_incremental re-queues # them instead of reading them as unchanged (#1948). @@ -588,7 +589,7 @@ _cleared = _dispatched - _stamped # files newly excluded since last run are dropped rather than masquerading as # deletions; untouched files' prior rows are still preserved (#1908). _scan = {f for fl in _corpus.values() for f in fl} -save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) # Update cumulative cost tracker input_tok = extract.get('input_tokens', 0) @@ -612,10 +613,10 @@ cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\" print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" "INPUT_PATH" +" rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null -rm -f graphify-out/needs_update 2>/dev/null || true +rm -f graphify-out/.needs_update 2>/dev/null || true ``` Replace INPUT_PATH with the actual path (same value used in Steps 4-5) so the manifest is relativized to the scan root. @@ -680,7 +681,7 @@ Both are non-default subcommands. `--update` re-extracts only new or changed fil When `graphify-out/graph.json` already exists and the user asks a question about the corpus, answer from the graph rather than rebuilding it: ```bash -$(cat graphify-out/.graphify_python) -m graphify query "" +graphify query "" ``` Before traversal, expand the question against the graph's own vocabulary so a wording mismatch does not collapse the answer to noise. If the `graphify query` CLI is unavailable, fall back to an inline NetworkX traversal of `graphify-out/graph.json`. Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. For that vocab-expansion step, the BFS/DFS traversal modes, the `--budget` cap, the NetworkX fallback, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`. diff --git a/graphify/skill-windows.md b/graphify/skill-windows.md index fbbf2277ab..b09ecca3c4 100644 --- a/graphify/skill-windows.md +++ b/graphify/skill-windows.md @@ -124,7 +124,7 @@ if (-not $GRAPHIFY_PYTHON) { $Utf8NoBom = New-Object System.Text.UTF8Encoding $false [System.IO.File]::WriteAllText((Join-Path $PWD 'graphify-out\.graphify_python'), [string]$GRAPHIFY_PYTHON, $Utf8NoBom) # Save scan root so `graphify update` (no args) knows where to look next time -[System.IO.File]::WriteAllText((Join-Path $PWD 'graphify-out\.graphify_root'), (Resolve-Path "INPUT_PATH").Path, $Utf8NoBom) +[System.IO.File]::WriteAllText((Join-Path $PWD 'graphify-out\.graphify_root'), (Resolve-Path INPUT_PATH).Path, $Utf8NoBom) ``` If the import succeeds, print nothing and move straight to Step 2. @@ -135,15 +135,15 @@ If the import succeeds, print nothing and move straight to Step 2. ```powershell @' -import sys, json +import json from graphify.detect import detect from pathlib import Path -result = detect(Path(sys.argv[1])) +result = detect(Path('INPUT_PATH')) # Write the sidecar from Python, not a shell redirect, so the same block renders # on PowerShell hosts without console-encoding drift (#2528). Path('graphify-out/.graphify_detect.json').write_text(json.dumps(result, ensure_ascii=False), encoding="utf-8") print(f'Detected {result["total_files"]} files') -'@ | & (Get-Content graphify-out\.graphify_python) - "INPUT_PATH" +'@ | & (Get-Content graphify-out\.graphify_python) - ``` Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: @@ -203,6 +203,7 @@ For any code files detected, run AST extraction in parallel with Part B subagent import sys, json from graphify.extract import collect_files, extract from pathlib import Path +import json code_files = [] detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding="utf-8")) @@ -210,13 +211,13 @@ for f in detect.get('files', {}).get('code', []): code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) if code_files: - result = extract(code_files, cache_root=Path(sys.argv[1])) + result = extract(code_files, cache_root=Path('INPUT_PATH')) Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding="utf-8") print(f'AST: {len(result["nodes"])} nodes, {len(result["edges"])} edges') else: Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding="utf-8") print('No code files - skipping AST extraction') -'@ | & (Get-Content graphify-out\.graphify_python) - "INPUT_PATH" +'@ | & (Get-Content graphify-out\.graphify_python) - ``` #### Part B - Semantic extraction (parallel subagents) @@ -231,7 +232,7 @@ Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],' '@ | & (Get-Content graphify-out\.graphify_python) - ``` -**MANDATORY: You MUST use the subagent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the subagent tool you are doing this wrong.** +**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** Before dispatching subagents, print a timing estimate: - Load `total_words` and file counts from `graphify-out/.graphify_detect.json` @@ -247,7 +248,7 @@ SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` ```powershell @' -import sys, json +import json from graphify.cache import check_semantic_cache from pathlib import Path @@ -257,7 +258,7 @@ detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encodin # every source file (#1392). Video is transcribed to a document in Step 2.5 first. all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root=sys.argv[1], prompt_file=sys.argv[2]) +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') # Always (re)write the cache file: write hits, else DELETE any leftover from a prior # run so Part C never merges a stale .graphify_cached.json (#1392). @@ -267,7 +268,7 @@ else: Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding="utf-8") print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -'@ | & (Get-Content graphify-out\.graphify_python) - "INPUT_PATH" "SPEC_PATH" +'@ | & (Get-Content graphify-out\.graphify_python) - ``` Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. @@ -312,7 +313,7 @@ Wait for all subagents. For each result: If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. -Merge all chunk files into `.graphify_semantic_new.json`. **After each subagent call completes, read the real token counts from the subagent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: +Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: ```powershell @' import json, glob @@ -339,15 +340,15 @@ print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens' Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): ```powershell @' -import sys, json +import json from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding="utf-8")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding="utf-8").splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root=sys.argv[1], allowed_source_files=uncached, prompt_file=sys.argv[2]) +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') print(f'Cached {saved} files') -'@ | & (Get-Content graphify-out\.graphify_python) - "INPUT_PATH" "SPEC_PATH" +'@ | & (Get-Content graphify-out\.graphify_python) - ``` Merge cached + new results into `graphify-out/.graphify_semantic.json`: @@ -436,7 +437,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -461,7 +462,7 @@ if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') print('If this shrink is intentional (you deleted files), re-run a full build with --force.') raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, sys.argv[1], suggested_questions=questions) +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding="utf-8") analysis = { 'communities': {str(k): v for k, v in communities.items()}, @@ -472,7 +473,7 @@ analysis = { } Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding="utf-8") print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -'@ | & (Get-Content graphify-out\.graphify_python) - "INPUT_PATH" +'@ | & (Get-Content graphify-out\.graphify_python) - ``` If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. @@ -485,12 +486,12 @@ A non-destructive diagnostic on the extraction, before labeling. It surfaces edg ```powershell @' -import sys, json +import json from pathlib import Path from graphify.diagnostics import diagnose_extraction, format_diagnostic_report extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding="utf-8")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root=sys.argv[1]) +summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( ('dangling_endpoint_edges', 'dangling-endpoint edges'), @@ -500,7 +501,7 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0)] print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') -'@ | & (Get-Content graphify-out\.graphify_python) - "INPUT_PATH" +'@ | & (Get-Content graphify-out\.graphify_python) - ``` Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules). @@ -526,7 +527,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding="utf-8")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} @@ -537,7 +538,9 @@ labels = LABELS_DICT # Regenerate questions with real community labels (labels affect question phrasing) questions = suggest_questions(G, communities, labels) -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, sys.argv[1], suggested_questions=questions) +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding="utf-8") +Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding="utf-8") # Re-export so graph.json nodes carry the curated community_name (#2490). # Same extraction as Step 4, so the #479 shrink-guard passes on node count; # if it still refuses, surface the guard message - do not force past it. @@ -545,11 +548,8 @@ wrote = to_json(G, communities, 'graphify-out/graph.json', community_labels=labe if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') print('If this shrink is intentional (you deleted files), re-run a full build with --force.') - raise SystemExit(1) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding="utf-8") -Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding="utf-8") print('Report updated with community labels') -'@ | & (Get-Content graphify-out\.graphify_python) - "INPUT_PATH" +'@ | & (Get-Content graphify-out\.graphify_python) - ``` Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). @@ -564,14 +564,15 @@ If `--obsidian` was given: - If `--obsidian-dir ` was also given, pass it via `--dir`. Otherwise defaults to `graphify-out/obsidian`. ```powershell -& (Get-Content graphify-out\.graphify_python) -m graphify export obsidian -# Add `--dir ~/vaults/my-project` to use a custom directory. +graphify export obsidian +# or with custom dir: graphify export obsidian --dir ~/vaults/my-project ``` Generate the HTML graph (always, unless `--no-viz`): ```powershell -& (Get-Content graphify-out\.graphify_python) -m graphify export html # auto-aggregates to community view if graph > 5000 nodes +graphify export html # auto-aggregates to community view if graph > 5000 nodes +# or: graphify export html --no-viz ``` ### Steps 6b-8 - Wiki, Neo4j, FalkorDB, SVG, GraphML, MCP, benchmark (only on their flags) @@ -584,7 +585,7 @@ These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, ` ```powershell @' -import sys, json +import json from pathlib import Path from datetime import datetime, timezone from graphify.detect import save_manifest @@ -607,7 +608,7 @@ extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encod # types are gated on output. from graphify.cli import _stamped_manifest_files _corpus = detect.get('all_files') or detect['files'] -_manifest_files = _stamped_manifest_files(_corpus, extract, Path(sys.argv[1])) +_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) # Files dispatched this run (the changed subset) but NOT stamped above still carry # a stale semantic_hash from a prior run; clear it so detect_incremental re-queues # them instead of reading them as unchanged (#1948). @@ -619,7 +620,7 @@ _cleared = _dispatched - _stamped # files newly excluded since last run are dropped rather than masquerading as # deletions; untouched files' prior rows are still preserved (#1908). _scan = {f for fl in _corpus.values() for f in fl} -save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) # Update cumulative cost tracker input_tok = extract.get('input_tokens', 0) @@ -643,10 +644,10 @@ cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding="u print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') print(f'All time: {cost["total_input_tokens"]:,} input, {cost["total_output_tokens"]:,} output ({len(cost["runs"])} runs)') -'@ | & (Get-Content graphify-out\.graphify_python) - "INPUT_PATH" +'@ | & (Get-Content graphify-out\.graphify_python) - Remove-Item -Force -ErrorAction SilentlyContinue graphify-out\.graphify_detect.json, graphify-out\.graphify_extract.json, graphify-out\.graphify_ast.json, graphify-out\.graphify_semantic.json, graphify-out\.graphify_analysis.json Get-ChildItem graphify-out -Filter '.graphify_chunk_*.json' -File -ErrorAction SilentlyContinue | Remove-Item -Force -Remove-Item -Force -ErrorAction SilentlyContinue graphify-out\needs_update +Remove-Item -Force -ErrorAction SilentlyContinue graphify-out\.needs_update ``` Replace INPUT_PATH with the actual path (same value used in Steps 4-5) so the manifest is relativized to the scan root. @@ -713,7 +714,7 @@ Both are non-default subcommands. `--update` re-extracts only new or changed fil When `graphify-out/graph.json` already exists and the user asks a question about the corpus, answer from the graph rather than rebuilding it: ```powershell -& (Get-Content graphify-out\.graphify_python) -m graphify query "" +graphify query "" ``` Before traversal, expand the question against the graph's own vocabulary so a wording mismatch does not collapse the answer to noise. If the `graphify query` CLI is unavailable, fall back to an inline NetworkX traversal of `graphify-out/graph.json`. Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. For that vocab-expansion step, the BFS/DFS traversal modes, the `--budget` cap, the NetworkX fallback, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`. diff --git a/graphify/skill.md b/graphify/skill.md index 55b77d6828..abd2811d23 100644 --- a/graphify/skill.md +++ b/graphify/skill.md @@ -97,7 +97,7 @@ fi mkdir -p graphify-out "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" # Save scan root so `graphify update` (no args) knows where to look next time -echo "$(cd "INPUT_PATH" && pwd)" > graphify-out/.graphify_root +echo "$(cd INPUT_PATH && pwd)" > graphify-out/.graphify_root ``` If the import succeeds, print nothing and move straight to Step 2. @@ -108,15 +108,15 @@ If the import succeeds, print nothing and move straight to Step 2. ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from graphify.detect import detect from pathlib import Path -result = detect(Path(sys.argv[1])) +result = detect(Path('INPUT_PATH')) # Write the sidecar from Python, not a shell redirect, so the same block renders # on PowerShell hosts without console-encoding drift (#2528). Path('graphify-out/.graphify_detect.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") print(f'Detected {result[\"total_files\"]} files') -" "INPUT_PATH" +" ``` Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: @@ -176,6 +176,7 @@ $(cat graphify-out/.graphify_python) -c " import sys, json from graphify.extract import collect_files, extract from pathlib import Path +import json code_files = [] detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) @@ -183,13 +184,13 @@ for f in detect.get('files', {}).get('code', []): code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) if code_files: - result = extract(code_files, cache_root=Path(sys.argv[1])) + result = extract(code_files, cache_root=Path('INPUT_PATH')) Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') else: Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") print('No code files - skipping AST extraction') -" "INPUT_PATH" +" ``` #### Part B - Semantic extraction (parallel subagents) @@ -204,7 +205,7 @@ Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],' " ``` -**MANDATORY: You MUST use the subagent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the subagent tool you are doing this wrong.** +**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** Before dispatching subagents, print a timing estimate: - Load `total_words` and file counts from `graphify-out/.graphify_detect.json` @@ -220,7 +221,7 @@ SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from graphify.cache import check_semantic_cache from pathlib import Path @@ -230,7 +231,7 @@ detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encodin # every source file (#1392). Video is transcribed to a document in Step 2.5 first. all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root=sys.argv[1], prompt_file=sys.argv[2]) +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') # Always (re)write the cache file: write hits, else DELETE any leftover from a prior # run so Part C never merges a stale .graphify_cached.json (#1392). @@ -240,7 +241,7 @@ else: Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" "INPUT_PATH" "SPEC_PATH" +" ``` Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. @@ -285,7 +286,7 @@ Wait for all subagents. For each result: If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. -Merge all chunk files into `.graphify_semantic_new.json`. **After each subagent call completes, read the real token counts from the subagent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: +Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: ```bash $(cat graphify-out/.graphify_python) -c " import json, glob @@ -312,15 +313,15 @@ print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens' Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root=sys.argv[1], allowed_source_files=uncached, prompt_file=sys.argv[2]) +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') print(f'Cached {saved} files') -" "INPUT_PATH" "SPEC_PATH" +" ``` Merge cached + new results into `graphify-out/.graphify_semantic.json`: @@ -409,7 +410,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -434,7 +435,7 @@ if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') print('If this shrink is intentional (you deleted files), re-run a full build with --force.') raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, sys.argv[1], suggested_questions=questions) +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") analysis = { 'communities': {str(k): v for k, v in communities.items()}, @@ -445,7 +446,7 @@ analysis = { } Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" "INPUT_PATH" +" ``` If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. @@ -458,12 +459,12 @@ A non-destructive diagnostic on the extraction, before labeling. It surfaces edg ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from pathlib import Path from graphify.diagnostics import diagnose_extraction, format_diagnostic_report extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root=sys.argv[1]) +summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( ('dangling_endpoint_edges', 'dangling-endpoint edges'), @@ -473,7 +474,7 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0)] print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') -" "INPUT_PATH" +" ``` Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules). @@ -499,7 +500,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} @@ -510,7 +511,9 @@ labels = LABELS_DICT # Regenerate questions with real community labels (labels affect question phrasing) questions = suggest_questions(G, communities, labels) -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, sys.argv[1], suggested_questions=questions) +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") +Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") # Re-export so graph.json nodes carry the curated community_name (#2490). # Same extraction as Step 4, so the #479 shrink-guard passes on node count; # if it still refuses, surface the guard message - do not force past it. @@ -518,11 +521,8 @@ wrote = to_json(G, communities, 'graphify-out/graph.json', community_labels=labe if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') print('If this shrink is intentional (you deleted files), re-run a full build with --force.') - raise SystemExit(1) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") print('Report updated with community labels') -" "INPUT_PATH" +" ``` Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). @@ -537,14 +537,15 @@ If `--obsidian` was given: - If `--obsidian-dir ` was also given, pass it via `--dir`. Otherwise defaults to `graphify-out/obsidian`. ```bash -$(cat graphify-out/.graphify_python) -m graphify export obsidian -# Add `--dir ~/vaults/my-project` to use a custom directory. +graphify export obsidian +# or with custom dir: graphify export obsidian --dir ~/vaults/my-project ``` Generate the HTML graph (always, unless `--no-viz`): ```bash -$(cat graphify-out/.graphify_python) -m graphify export html # auto-aggregates to community view if graph > 5000 nodes +graphify export html # auto-aggregates to community view if graph > 5000 nodes +# or: graphify export html --no-viz ``` ### Steps 6b-8 - Wiki, Neo4j, FalkorDB, SVG, GraphML, MCP, benchmark (only on their flags) @@ -557,7 +558,7 @@ These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, ` ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from pathlib import Path from datetime import datetime, timezone from graphify.detect import save_manifest @@ -580,7 +581,7 @@ extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encod # types are gated on output. from graphify.cli import _stamped_manifest_files _corpus = detect.get('all_files') or detect['files'] -_manifest_files = _stamped_manifest_files(_corpus, extract, Path(sys.argv[1])) +_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) # Files dispatched this run (the changed subset) but NOT stamped above still carry # a stale semantic_hash from a prior run; clear it so detect_incremental re-queues # them instead of reading them as unchanged (#1948). @@ -592,7 +593,7 @@ _cleared = _dispatched - _stamped # files newly excluded since last run are dropped rather than masquerading as # deletions; untouched files' prior rows are still preserved (#1908). _scan = {f for fl in _corpus.values() for f in fl} -save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) # Update cumulative cost tracker input_tok = extract.get('input_tokens', 0) @@ -616,10 +617,10 @@ cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\" print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" "INPUT_PATH" +" rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null -rm -f graphify-out/needs_update 2>/dev/null || true +rm -f graphify-out/.needs_update 2>/dev/null || true ``` Replace INPUT_PATH with the actual path (same value used in Steps 4-5) so the manifest is relativized to the scan root. @@ -684,7 +685,7 @@ Both are non-default subcommands. `--update` re-extracts only new or changed fil When `graphify-out/graph.json` already exists and the user asks a question about the corpus, answer from the graph rather than rebuilding it: ```bash -$(cat graphify-out/.graphify_python) -m graphify query "" +graphify query "" ``` Before traversal, expand the question against the graph's own vocabulary so a wording mismatch does not collapse the answer to noise. If the `graphify query` CLI is unavailable, fall back to an inline NetworkX traversal of `graphify-out/graph.json`. Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. For that vocab-expansion step, the BFS/DFS traversal modes, the `--budget` cap, the NetworkX fallback, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`. diff --git a/graphify/skills/agents/references/add-watch.md b/graphify/skills/agents/references/add-watch.md index fcbe0cd488..77844343e1 100644 --- a/graphify/skills/agents/references/add-watch.md +++ b/graphify/skills/agents/references/add-watch.md @@ -13,7 +13,7 @@ from graphify.ingest import ingest from pathlib import Path try: - out = ingest(sys.argv[1], Path('./raw'), author=sys.argv[2] or None, contributor=sys.argv[3] or None) + out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') print(f'Saved to {out}') except ValueError as e: print(f'error: {e}', file=sys.stderr) @@ -21,10 +21,10 @@ except ValueError as e: except RuntimeError as e: print(f'error: {e}', file=sys.stderr) sys.exit(1) -" "URL" "AUTHOR" "CONTRIBUTOR" +" ``` -Replace `URL` with the actual URL and pass the user's `AUTHOR` or `CONTRIBUTOR` when provided; use an empty string for either omitted value. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on the scan root recorded in `graphify-out/.graphify_root` to merge the new file into the existing graph. +Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. Supported URL types (auto-detected): - YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`) @@ -41,7 +41,7 @@ Supported URL types (auto-detected): Start a background watcher that monitors a folder and auto-updates the graph when files change. ```bash -$(cat graphify-out/.graphify_python) -m graphify.watch "INPUT_PATH" --debounce 3 +$(cat graphify-out/.graphify_python) -m graphify.watch INPUT_PATH --debounce 3 ``` Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: diff --git a/graphify/skills/agents/references/exports.md b/graphify/skills/agents/references/exports.md index 89ce996fb7..242ff868e0 100644 --- a/graphify/skills/agents/references/exports.md +++ b/graphify/skills/agents/references/exports.md @@ -20,27 +20,13 @@ graphify export wiki graphify export neo4j ``` -**If `--neo4j-push `** - push directly to a running Neo4j instance. Do not run these blocks through an agent tool: its shell is non-interactive and cannot read a hidden password. Show the block for the user's shell, ask them to run it in their own terminal, and wait for confirmation before continuing. Never ask them to paste the password into chat. - -POSIX shell: +**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: ```bash -NEO4J_PASSWORD="$("$(cat graphify-out/.graphify_python)" -c 'import getpass; print(getpass.getpass("Neo4j password: "))')" -export NEO4J_PASSWORD -graphify export neo4j --push bolt://localhost:7687 --user neo4j -unset NEO4J_PASSWORD -``` - -PowerShell: - -```powershell -$credential = Read-Host "Neo4j password" -AsSecureString -$env:NEO4J_PASSWORD = [System.Net.NetworkCredential]::new("", $credential).Password -graphify export neo4j --push bolt://localhost:7687 --user neo4j -Remove-Item Env:NEO4J_PASSWORD +graphify export neo4j --push bolt://localhost:7687 --user neo4j --password PASSWORD ``` -Default URI is `bolt://localhost:7687`, default user is `neo4j`. The command reads the password from `NEO4J_PASSWORD`. Uses MERGE - safe to re-run without creating duplicates. +Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. ### Step 7a - FalkorDB export (only if --falkordb or --falkordb-push flag) diff --git a/graphify/skills/agents/references/extraction-spec.md b/graphify/skills/agents/references/extraction-spec.md index a5a12a575e..388df7674f 100644 --- a/graphify/skills/agents/references/extraction-spec.md +++ b/graphify/skills/agents/references/extraction-spec.md @@ -61,7 +61,7 @@ confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a d Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is the **full repo-relative path with the extension dropped**, every path segment kept and joined with `_` (each segment lowercased with non-alphanumeric chars replaced by `_`), and entity is the symbol name similarly normalized. Use every directory level, not just the immediate parent — this keeps same-named files in different directories distinct. Examples: `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `lib_utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`; `docs/v1/api/README.md` + `getUser` → `docs_v1_api_readme_getuser`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or only the immediate parent (e.g., `auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project built under the old immediate-parent format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. Generate the extraction JSON matching this schema exactly: -{"nodes":[{"id":"src_auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} +{"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. diff --git a/graphify/skills/agents/references/query.md b/graphify/skills/agents/references/query.md index 6dd627863c..56565eb782 100644 --- a/graphify/skills/agents/references/query.md +++ b/graphify/skills/agents/references/query.md @@ -79,12 +79,12 @@ If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal ```bash $(cat graphify-out/.graphify_python) -c " import sys, json -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph import networkx as nx from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' @@ -197,11 +197,11 @@ If the CLI is unavailable, run it inline: $(cat graphify-out/.graphify_python) -c " import json, sys import networkx as nx -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') a_term = 'NODE_A' b_term = 'NODE_B' @@ -265,11 +265,11 @@ If the CLI is unavailable, run it inline: $(cat graphify-out/.graphify_python) -c " import json, sys import networkx as nx -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') term = 'NODE_NAME' term_lower = term.lower() diff --git a/graphify/skills/agents/references/transcribe.md b/graphify/skills/agents/references/transcribe.md index bbe8b93e45..b967f83799 100644 --- a/graphify/skills/agents/references/transcribe.md +++ b/graphify/skills/agents/references/transcribe.md @@ -8,11 +8,13 @@ Skip this step entirely if `detect` returned zero `video` files. Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. -**Strategy:** If `graphify-out/.graphify_analysis.json` exists from a previous run, read its top god-node labels and write a one-sentence domain hint from them. Otherwise use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` Pass the chosen prompt to Whisper as its initial prompt. No separate API call needed. +**Strategy:** Read the god nodes from `graphify-out/.graphify_detect.json` (or the analysis file if it exists from a previous run). You are already a language model — write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. + +**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` **Step 1 - Write the Whisper prompt yourself.** -Read the top god-node labels from previous analysis when available, then compose a short domain hint sentence. With no previous analysis, use the generic fallback above. +Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: - Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` - Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` diff --git a/graphify/skills/agents/references/update.md b/graphify/skills/agents/references/update.md index 058ca8e6d2..3632fd4126 100644 --- a/graphify/skills/agents/references/update.md +++ b/graphify/skills/agents/references/update.md @@ -12,7 +12,7 @@ import sys, json from graphify.detect import detect_incremental, save_manifest from pathlib import Path -result = detect_incremental(Path(sys.argv[1])) +result = detect_incremental(Path('INPUT_PATH')) new_total = result.get('new_total', 0) print(json.dumps(result, indent=2, ensure_ascii=False)) Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") @@ -24,7 +24,7 @@ if deleted: print(f'{len(deleted)} deleted file(s) to prune.') if new_total > 0: print(f'{new_total} new/changed file(s) to re-extract.') -" "INPUT_PATH" +" ``` Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: @@ -84,7 +84,7 @@ Then: ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from pathlib import Path from graphify.build import build_merge from graphify.detect import save_manifest @@ -113,7 +113,7 @@ G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, - root=sys.argv[1], + root='INPUT_PATH', directed=IS_DIRECTED, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -150,7 +150,7 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) # is lost forever (#2015). Mirrors the library extract path # (cli._stamped_manifest_files + clear_semantic + scan_corpus). from graphify.cli import _stamped_manifest_files -_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path(sys.argv[1])) +_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) # Changed semantic files dispatched this run but NOT stamped had their chunk fail # or be omitted; clear any stale semantic_hash so they are re-queued (#1948). _sem_types = ('document', 'paper', 'image') @@ -160,9 +160,9 @@ _cleared = _dispatched - _stamped # scan_corpus = the RAW full corpus so in-root files newly excluded since last run # are dropped rather than masquerading as deletions; untouched rows preserved (#1908). _scan = {f for fl in incremental['files'].values() for f in fl} -save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) print('[graphify update] Manifest saved.') -" "INPUT_PATH" +" ``` Then run Steps 4–8 on the merged graph as normal. @@ -174,7 +174,7 @@ $(cat graphify-out/.graphify_python) -c " import json from graphify.analyze import graph_diff from graphify.build import build_from_json -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph import networkx as nx from pathlib import Path @@ -184,7 +184,7 @@ new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(e G_new = build_from_json(new_extract, directed=IS_DIRECTED) if old_data: - G_old = load_node_link_graph(old_data) + G_old = json_graph.node_link_graph(old_data, edges='links') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: diff --git a/graphify/skills/amp/references/add-watch.md b/graphify/skills/amp/references/add-watch.md index fcbe0cd488..77844343e1 100644 --- a/graphify/skills/amp/references/add-watch.md +++ b/graphify/skills/amp/references/add-watch.md @@ -13,7 +13,7 @@ from graphify.ingest import ingest from pathlib import Path try: - out = ingest(sys.argv[1], Path('./raw'), author=sys.argv[2] or None, contributor=sys.argv[3] or None) + out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') print(f'Saved to {out}') except ValueError as e: print(f'error: {e}', file=sys.stderr) @@ -21,10 +21,10 @@ except ValueError as e: except RuntimeError as e: print(f'error: {e}', file=sys.stderr) sys.exit(1) -" "URL" "AUTHOR" "CONTRIBUTOR" +" ``` -Replace `URL` with the actual URL and pass the user's `AUTHOR` or `CONTRIBUTOR` when provided; use an empty string for either omitted value. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on the scan root recorded in `graphify-out/.graphify_root` to merge the new file into the existing graph. +Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. Supported URL types (auto-detected): - YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`) @@ -41,7 +41,7 @@ Supported URL types (auto-detected): Start a background watcher that monitors a folder and auto-updates the graph when files change. ```bash -$(cat graphify-out/.graphify_python) -m graphify.watch "INPUT_PATH" --debounce 3 +$(cat graphify-out/.graphify_python) -m graphify.watch INPUT_PATH --debounce 3 ``` Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: diff --git a/graphify/skills/amp/references/exports.md b/graphify/skills/amp/references/exports.md index 89ce996fb7..242ff868e0 100644 --- a/graphify/skills/amp/references/exports.md +++ b/graphify/skills/amp/references/exports.md @@ -20,27 +20,13 @@ graphify export wiki graphify export neo4j ``` -**If `--neo4j-push `** - push directly to a running Neo4j instance. Do not run these blocks through an agent tool: its shell is non-interactive and cannot read a hidden password. Show the block for the user's shell, ask them to run it in their own terminal, and wait for confirmation before continuing. Never ask them to paste the password into chat. - -POSIX shell: +**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: ```bash -NEO4J_PASSWORD="$("$(cat graphify-out/.graphify_python)" -c 'import getpass; print(getpass.getpass("Neo4j password: "))')" -export NEO4J_PASSWORD -graphify export neo4j --push bolt://localhost:7687 --user neo4j -unset NEO4J_PASSWORD -``` - -PowerShell: - -```powershell -$credential = Read-Host "Neo4j password" -AsSecureString -$env:NEO4J_PASSWORD = [System.Net.NetworkCredential]::new("", $credential).Password -graphify export neo4j --push bolt://localhost:7687 --user neo4j -Remove-Item Env:NEO4J_PASSWORD +graphify export neo4j --push bolt://localhost:7687 --user neo4j --password PASSWORD ``` -Default URI is `bolt://localhost:7687`, default user is `neo4j`. The command reads the password from `NEO4J_PASSWORD`. Uses MERGE - safe to re-run without creating duplicates. +Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. ### Step 7a - FalkorDB export (only if --falkordb or --falkordb-push flag) diff --git a/graphify/skills/amp/references/extraction-spec.md b/graphify/skills/amp/references/extraction-spec.md index a5a12a575e..388df7674f 100644 --- a/graphify/skills/amp/references/extraction-spec.md +++ b/graphify/skills/amp/references/extraction-spec.md @@ -61,7 +61,7 @@ confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a d Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is the **full repo-relative path with the extension dropped**, every path segment kept and joined with `_` (each segment lowercased with non-alphanumeric chars replaced by `_`), and entity is the symbol name similarly normalized. Use every directory level, not just the immediate parent — this keeps same-named files in different directories distinct. Examples: `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `lib_utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`; `docs/v1/api/README.md` + `getUser` → `docs_v1_api_readme_getuser`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or only the immediate parent (e.g., `auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project built under the old immediate-parent format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. Generate the extraction JSON matching this schema exactly: -{"nodes":[{"id":"src_auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} +{"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. diff --git a/graphify/skills/amp/references/query.md b/graphify/skills/amp/references/query.md index 6dd627863c..56565eb782 100644 --- a/graphify/skills/amp/references/query.md +++ b/graphify/skills/amp/references/query.md @@ -79,12 +79,12 @@ If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal ```bash $(cat graphify-out/.graphify_python) -c " import sys, json -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph import networkx as nx from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' @@ -197,11 +197,11 @@ If the CLI is unavailable, run it inline: $(cat graphify-out/.graphify_python) -c " import json, sys import networkx as nx -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') a_term = 'NODE_A' b_term = 'NODE_B' @@ -265,11 +265,11 @@ If the CLI is unavailable, run it inline: $(cat graphify-out/.graphify_python) -c " import json, sys import networkx as nx -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') term = 'NODE_NAME' term_lower = term.lower() diff --git a/graphify/skills/amp/references/transcribe.md b/graphify/skills/amp/references/transcribe.md index bbe8b93e45..b967f83799 100644 --- a/graphify/skills/amp/references/transcribe.md +++ b/graphify/skills/amp/references/transcribe.md @@ -8,11 +8,13 @@ Skip this step entirely if `detect` returned zero `video` files. Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. -**Strategy:** If `graphify-out/.graphify_analysis.json` exists from a previous run, read its top god-node labels and write a one-sentence domain hint from them. Otherwise use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` Pass the chosen prompt to Whisper as its initial prompt. No separate API call needed. +**Strategy:** Read the god nodes from `graphify-out/.graphify_detect.json` (or the analysis file if it exists from a previous run). You are already a language model — write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. + +**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` **Step 1 - Write the Whisper prompt yourself.** -Read the top god-node labels from previous analysis when available, then compose a short domain hint sentence. With no previous analysis, use the generic fallback above. +Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: - Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` - Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` diff --git a/graphify/skills/amp/references/update.md b/graphify/skills/amp/references/update.md index 058ca8e6d2..3632fd4126 100644 --- a/graphify/skills/amp/references/update.md +++ b/graphify/skills/amp/references/update.md @@ -12,7 +12,7 @@ import sys, json from graphify.detect import detect_incremental, save_manifest from pathlib import Path -result = detect_incremental(Path(sys.argv[1])) +result = detect_incremental(Path('INPUT_PATH')) new_total = result.get('new_total', 0) print(json.dumps(result, indent=2, ensure_ascii=False)) Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") @@ -24,7 +24,7 @@ if deleted: print(f'{len(deleted)} deleted file(s) to prune.') if new_total > 0: print(f'{new_total} new/changed file(s) to re-extract.') -" "INPUT_PATH" +" ``` Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: @@ -84,7 +84,7 @@ Then: ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from pathlib import Path from graphify.build import build_merge from graphify.detect import save_manifest @@ -113,7 +113,7 @@ G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, - root=sys.argv[1], + root='INPUT_PATH', directed=IS_DIRECTED, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -150,7 +150,7 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) # is lost forever (#2015). Mirrors the library extract path # (cli._stamped_manifest_files + clear_semantic + scan_corpus). from graphify.cli import _stamped_manifest_files -_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path(sys.argv[1])) +_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) # Changed semantic files dispatched this run but NOT stamped had their chunk fail # or be omitted; clear any stale semantic_hash so they are re-queued (#1948). _sem_types = ('document', 'paper', 'image') @@ -160,9 +160,9 @@ _cleared = _dispatched - _stamped # scan_corpus = the RAW full corpus so in-root files newly excluded since last run # are dropped rather than masquerading as deletions; untouched rows preserved (#1908). _scan = {f for fl in incremental['files'].values() for f in fl} -save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) print('[graphify update] Manifest saved.') -" "INPUT_PATH" +" ``` Then run Steps 4–8 on the merged graph as normal. @@ -174,7 +174,7 @@ $(cat graphify-out/.graphify_python) -c " import json from graphify.analyze import graph_diff from graphify.build import build_from_json -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph import networkx as nx from pathlib import Path @@ -184,7 +184,7 @@ new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(e G_new = build_from_json(new_extract, directed=IS_DIRECTED) if old_data: - G_old = load_node_link_graph(old_data) + G_old = json_graph.node_link_graph(old_data, edges='links') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: diff --git a/graphify/skills/claude/references/add-watch.md b/graphify/skills/claude/references/add-watch.md index fcbe0cd488..77844343e1 100644 --- a/graphify/skills/claude/references/add-watch.md +++ b/graphify/skills/claude/references/add-watch.md @@ -13,7 +13,7 @@ from graphify.ingest import ingest from pathlib import Path try: - out = ingest(sys.argv[1], Path('./raw'), author=sys.argv[2] or None, contributor=sys.argv[3] or None) + out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') print(f'Saved to {out}') except ValueError as e: print(f'error: {e}', file=sys.stderr) @@ -21,10 +21,10 @@ except ValueError as e: except RuntimeError as e: print(f'error: {e}', file=sys.stderr) sys.exit(1) -" "URL" "AUTHOR" "CONTRIBUTOR" +" ``` -Replace `URL` with the actual URL and pass the user's `AUTHOR` or `CONTRIBUTOR` when provided; use an empty string for either omitted value. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on the scan root recorded in `graphify-out/.graphify_root` to merge the new file into the existing graph. +Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. Supported URL types (auto-detected): - YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`) @@ -41,7 +41,7 @@ Supported URL types (auto-detected): Start a background watcher that monitors a folder and auto-updates the graph when files change. ```bash -$(cat graphify-out/.graphify_python) -m graphify.watch "INPUT_PATH" --debounce 3 +$(cat graphify-out/.graphify_python) -m graphify.watch INPUT_PATH --debounce 3 ``` Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: diff --git a/graphify/skills/claude/references/exports.md b/graphify/skills/claude/references/exports.md index 89ce996fb7..242ff868e0 100644 --- a/graphify/skills/claude/references/exports.md +++ b/graphify/skills/claude/references/exports.md @@ -20,27 +20,13 @@ graphify export wiki graphify export neo4j ``` -**If `--neo4j-push `** - push directly to a running Neo4j instance. Do not run these blocks through an agent tool: its shell is non-interactive and cannot read a hidden password. Show the block for the user's shell, ask them to run it in their own terminal, and wait for confirmation before continuing. Never ask them to paste the password into chat. - -POSIX shell: +**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: ```bash -NEO4J_PASSWORD="$("$(cat graphify-out/.graphify_python)" -c 'import getpass; print(getpass.getpass("Neo4j password: "))')" -export NEO4J_PASSWORD -graphify export neo4j --push bolt://localhost:7687 --user neo4j -unset NEO4J_PASSWORD -``` - -PowerShell: - -```powershell -$credential = Read-Host "Neo4j password" -AsSecureString -$env:NEO4J_PASSWORD = [System.Net.NetworkCredential]::new("", $credential).Password -graphify export neo4j --push bolt://localhost:7687 --user neo4j -Remove-Item Env:NEO4J_PASSWORD +graphify export neo4j --push bolt://localhost:7687 --user neo4j --password PASSWORD ``` -Default URI is `bolt://localhost:7687`, default user is `neo4j`. The command reads the password from `NEO4J_PASSWORD`. Uses MERGE - safe to re-run without creating duplicates. +Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. ### Step 7a - FalkorDB export (only if --falkordb or --falkordb-push flag) diff --git a/graphify/skills/claude/references/extraction-spec.md b/graphify/skills/claude/references/extraction-spec.md index a5a12a575e..388df7674f 100644 --- a/graphify/skills/claude/references/extraction-spec.md +++ b/graphify/skills/claude/references/extraction-spec.md @@ -61,7 +61,7 @@ confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a d Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is the **full repo-relative path with the extension dropped**, every path segment kept and joined with `_` (each segment lowercased with non-alphanumeric chars replaced by `_`), and entity is the symbol name similarly normalized. Use every directory level, not just the immediate parent — this keeps same-named files in different directories distinct. Examples: `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `lib_utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`; `docs/v1/api/README.md` + `getUser` → `docs_v1_api_readme_getuser`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or only the immediate parent (e.g., `auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project built under the old immediate-parent format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. Generate the extraction JSON matching this schema exactly: -{"nodes":[{"id":"src_auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} +{"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. diff --git a/graphify/skills/claude/references/query.md b/graphify/skills/claude/references/query.md index 6dd627863c..56565eb782 100644 --- a/graphify/skills/claude/references/query.md +++ b/graphify/skills/claude/references/query.md @@ -79,12 +79,12 @@ If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal ```bash $(cat graphify-out/.graphify_python) -c " import sys, json -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph import networkx as nx from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' @@ -197,11 +197,11 @@ If the CLI is unavailable, run it inline: $(cat graphify-out/.graphify_python) -c " import json, sys import networkx as nx -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') a_term = 'NODE_A' b_term = 'NODE_B' @@ -265,11 +265,11 @@ If the CLI is unavailable, run it inline: $(cat graphify-out/.graphify_python) -c " import json, sys import networkx as nx -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') term = 'NODE_NAME' term_lower = term.lower() diff --git a/graphify/skills/claude/references/transcribe.md b/graphify/skills/claude/references/transcribe.md index bbe8b93e45..b967f83799 100644 --- a/graphify/skills/claude/references/transcribe.md +++ b/graphify/skills/claude/references/transcribe.md @@ -8,11 +8,13 @@ Skip this step entirely if `detect` returned zero `video` files. Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. -**Strategy:** If `graphify-out/.graphify_analysis.json` exists from a previous run, read its top god-node labels and write a one-sentence domain hint from them. Otherwise use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` Pass the chosen prompt to Whisper as its initial prompt. No separate API call needed. +**Strategy:** Read the god nodes from `graphify-out/.graphify_detect.json` (or the analysis file if it exists from a previous run). You are already a language model — write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. + +**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` **Step 1 - Write the Whisper prompt yourself.** -Read the top god-node labels from previous analysis when available, then compose a short domain hint sentence. With no previous analysis, use the generic fallback above. +Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: - Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` - Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` diff --git a/graphify/skills/claude/references/update.md b/graphify/skills/claude/references/update.md index 058ca8e6d2..3632fd4126 100644 --- a/graphify/skills/claude/references/update.md +++ b/graphify/skills/claude/references/update.md @@ -12,7 +12,7 @@ import sys, json from graphify.detect import detect_incremental, save_manifest from pathlib import Path -result = detect_incremental(Path(sys.argv[1])) +result = detect_incremental(Path('INPUT_PATH')) new_total = result.get('new_total', 0) print(json.dumps(result, indent=2, ensure_ascii=False)) Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") @@ -24,7 +24,7 @@ if deleted: print(f'{len(deleted)} deleted file(s) to prune.') if new_total > 0: print(f'{new_total} new/changed file(s) to re-extract.') -" "INPUT_PATH" +" ``` Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: @@ -84,7 +84,7 @@ Then: ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from pathlib import Path from graphify.build import build_merge from graphify.detect import save_manifest @@ -113,7 +113,7 @@ G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, - root=sys.argv[1], + root='INPUT_PATH', directed=IS_DIRECTED, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -150,7 +150,7 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) # is lost forever (#2015). Mirrors the library extract path # (cli._stamped_manifest_files + clear_semantic + scan_corpus). from graphify.cli import _stamped_manifest_files -_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path(sys.argv[1])) +_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) # Changed semantic files dispatched this run but NOT stamped had their chunk fail # or be omitted; clear any stale semantic_hash so they are re-queued (#1948). _sem_types = ('document', 'paper', 'image') @@ -160,9 +160,9 @@ _cleared = _dispatched - _stamped # scan_corpus = the RAW full corpus so in-root files newly excluded since last run # are dropped rather than masquerading as deletions; untouched rows preserved (#1908). _scan = {f for fl in incremental['files'].values() for f in fl} -save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) print('[graphify update] Manifest saved.') -" "INPUT_PATH" +" ``` Then run Steps 4–8 on the merged graph as normal. @@ -174,7 +174,7 @@ $(cat graphify-out/.graphify_python) -c " import json from graphify.analyze import graph_diff from graphify.build import build_from_json -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph import networkx as nx from pathlib import Path @@ -184,7 +184,7 @@ new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(e G_new = build_from_json(new_extract, directed=IS_DIRECTED) if old_data: - G_old = load_node_link_graph(old_data) + G_old = json_graph.node_link_graph(old_data, edges='links') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: diff --git a/graphify/skills/claw/references/add-watch.md b/graphify/skills/claw/references/add-watch.md index fcbe0cd488..77844343e1 100644 --- a/graphify/skills/claw/references/add-watch.md +++ b/graphify/skills/claw/references/add-watch.md @@ -13,7 +13,7 @@ from graphify.ingest import ingest from pathlib import Path try: - out = ingest(sys.argv[1], Path('./raw'), author=sys.argv[2] or None, contributor=sys.argv[3] or None) + out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') print(f'Saved to {out}') except ValueError as e: print(f'error: {e}', file=sys.stderr) @@ -21,10 +21,10 @@ except ValueError as e: except RuntimeError as e: print(f'error: {e}', file=sys.stderr) sys.exit(1) -" "URL" "AUTHOR" "CONTRIBUTOR" +" ``` -Replace `URL` with the actual URL and pass the user's `AUTHOR` or `CONTRIBUTOR` when provided; use an empty string for either omitted value. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on the scan root recorded in `graphify-out/.graphify_root` to merge the new file into the existing graph. +Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. Supported URL types (auto-detected): - YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`) @@ -41,7 +41,7 @@ Supported URL types (auto-detected): Start a background watcher that monitors a folder and auto-updates the graph when files change. ```bash -$(cat graphify-out/.graphify_python) -m graphify.watch "INPUT_PATH" --debounce 3 +$(cat graphify-out/.graphify_python) -m graphify.watch INPUT_PATH --debounce 3 ``` Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: diff --git a/graphify/skills/claw/references/exports.md b/graphify/skills/claw/references/exports.md index 89ce996fb7..242ff868e0 100644 --- a/graphify/skills/claw/references/exports.md +++ b/graphify/skills/claw/references/exports.md @@ -20,27 +20,13 @@ graphify export wiki graphify export neo4j ``` -**If `--neo4j-push `** - push directly to a running Neo4j instance. Do not run these blocks through an agent tool: its shell is non-interactive and cannot read a hidden password. Show the block for the user's shell, ask them to run it in their own terminal, and wait for confirmation before continuing. Never ask them to paste the password into chat. - -POSIX shell: +**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: ```bash -NEO4J_PASSWORD="$("$(cat graphify-out/.graphify_python)" -c 'import getpass; print(getpass.getpass("Neo4j password: "))')" -export NEO4J_PASSWORD -graphify export neo4j --push bolt://localhost:7687 --user neo4j -unset NEO4J_PASSWORD -``` - -PowerShell: - -```powershell -$credential = Read-Host "Neo4j password" -AsSecureString -$env:NEO4J_PASSWORD = [System.Net.NetworkCredential]::new("", $credential).Password -graphify export neo4j --push bolt://localhost:7687 --user neo4j -Remove-Item Env:NEO4J_PASSWORD +graphify export neo4j --push bolt://localhost:7687 --user neo4j --password PASSWORD ``` -Default URI is `bolt://localhost:7687`, default user is `neo4j`. The command reads the password from `NEO4J_PASSWORD`. Uses MERGE - safe to re-run without creating duplicates. +Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. ### Step 7a - FalkorDB export (only if --falkordb or --falkordb-push flag) diff --git a/graphify/skills/claw/references/extraction-spec.md b/graphify/skills/claw/references/extraction-spec.md index 8aaa23729b..4b278b28d3 100644 --- a/graphify/skills/claw/references/extraction-spec.md +++ b/graphify/skills/claw/references/extraction-spec.md @@ -25,7 +25,7 @@ Rules: Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format `{stem}_{entity}` where stem is the full repo-relative path with the extension dropped, every segment joined with `_` (each lowercased with non-alphanumeric chars replaced by `_`) and entity is the symbol name similarly normalized. Use every directory level, not just the immediate parent. `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`. Top-level files use just the filename stem. This must match the AST extractor's ID. Never append chunk or sequence suffixes — IDs must be deterministic from the label alone. Output exactly this JSON (no other text): -{"nodes":[{"id":"src_auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} +{"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} source_file RULE: set source_file to the FILE_LIST path for that file VERBATIM (absolute, no shortening to basename, no re-relativizing, no separator change). Keeps full build and --update on one base so build_merge's replace matches instead of duplicating. ``` diff --git a/graphify/skills/claw/references/query.md b/graphify/skills/claw/references/query.md index 6dd627863c..56565eb782 100644 --- a/graphify/skills/claw/references/query.md +++ b/graphify/skills/claw/references/query.md @@ -79,12 +79,12 @@ If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal ```bash $(cat graphify-out/.graphify_python) -c " import sys, json -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph import networkx as nx from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' @@ -197,11 +197,11 @@ If the CLI is unavailable, run it inline: $(cat graphify-out/.graphify_python) -c " import json, sys import networkx as nx -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') a_term = 'NODE_A' b_term = 'NODE_B' @@ -265,11 +265,11 @@ If the CLI is unavailable, run it inline: $(cat graphify-out/.graphify_python) -c " import json, sys import networkx as nx -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') term = 'NODE_NAME' term_lower = term.lower() diff --git a/graphify/skills/claw/references/transcribe.md b/graphify/skills/claw/references/transcribe.md index bbe8b93e45..b967f83799 100644 --- a/graphify/skills/claw/references/transcribe.md +++ b/graphify/skills/claw/references/transcribe.md @@ -8,11 +8,13 @@ Skip this step entirely if `detect` returned zero `video` files. Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. -**Strategy:** If `graphify-out/.graphify_analysis.json` exists from a previous run, read its top god-node labels and write a one-sentence domain hint from them. Otherwise use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` Pass the chosen prompt to Whisper as its initial prompt. No separate API call needed. +**Strategy:** Read the god nodes from `graphify-out/.graphify_detect.json` (or the analysis file if it exists from a previous run). You are already a language model — write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. + +**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` **Step 1 - Write the Whisper prompt yourself.** -Read the top god-node labels from previous analysis when available, then compose a short domain hint sentence. With no previous analysis, use the generic fallback above. +Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: - Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` - Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` diff --git a/graphify/skills/claw/references/update.md b/graphify/skills/claw/references/update.md index 058ca8e6d2..3632fd4126 100644 --- a/graphify/skills/claw/references/update.md +++ b/graphify/skills/claw/references/update.md @@ -12,7 +12,7 @@ import sys, json from graphify.detect import detect_incremental, save_manifest from pathlib import Path -result = detect_incremental(Path(sys.argv[1])) +result = detect_incremental(Path('INPUT_PATH')) new_total = result.get('new_total', 0) print(json.dumps(result, indent=2, ensure_ascii=False)) Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") @@ -24,7 +24,7 @@ if deleted: print(f'{len(deleted)} deleted file(s) to prune.') if new_total > 0: print(f'{new_total} new/changed file(s) to re-extract.') -" "INPUT_PATH" +" ``` Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: @@ -84,7 +84,7 @@ Then: ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from pathlib import Path from graphify.build import build_merge from graphify.detect import save_manifest @@ -113,7 +113,7 @@ G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, - root=sys.argv[1], + root='INPUT_PATH', directed=IS_DIRECTED, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -150,7 +150,7 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) # is lost forever (#2015). Mirrors the library extract path # (cli._stamped_manifest_files + clear_semantic + scan_corpus). from graphify.cli import _stamped_manifest_files -_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path(sys.argv[1])) +_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) # Changed semantic files dispatched this run but NOT stamped had their chunk fail # or be omitted; clear any stale semantic_hash so they are re-queued (#1948). _sem_types = ('document', 'paper', 'image') @@ -160,9 +160,9 @@ _cleared = _dispatched - _stamped # scan_corpus = the RAW full corpus so in-root files newly excluded since last run # are dropped rather than masquerading as deletions; untouched rows preserved (#1908). _scan = {f for fl in incremental['files'].values() for f in fl} -save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) print('[graphify update] Manifest saved.') -" "INPUT_PATH" +" ``` Then run Steps 4–8 on the merged graph as normal. @@ -174,7 +174,7 @@ $(cat graphify-out/.graphify_python) -c " import json from graphify.analyze import graph_diff from graphify.build import build_from_json -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph import networkx as nx from pathlib import Path @@ -184,7 +184,7 @@ new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(e G_new = build_from_json(new_extract, directed=IS_DIRECTED) if old_data: - G_old = load_node_link_graph(old_data) + G_old = json_graph.node_link_graph(old_data, edges='links') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: diff --git a/graphify/skills/codex/references/add-watch.md b/graphify/skills/codex/references/add-watch.md index fcbe0cd488..77844343e1 100644 --- a/graphify/skills/codex/references/add-watch.md +++ b/graphify/skills/codex/references/add-watch.md @@ -13,7 +13,7 @@ from graphify.ingest import ingest from pathlib import Path try: - out = ingest(sys.argv[1], Path('./raw'), author=sys.argv[2] or None, contributor=sys.argv[3] or None) + out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') print(f'Saved to {out}') except ValueError as e: print(f'error: {e}', file=sys.stderr) @@ -21,10 +21,10 @@ except ValueError as e: except RuntimeError as e: print(f'error: {e}', file=sys.stderr) sys.exit(1) -" "URL" "AUTHOR" "CONTRIBUTOR" +" ``` -Replace `URL` with the actual URL and pass the user's `AUTHOR` or `CONTRIBUTOR` when provided; use an empty string for either omitted value. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on the scan root recorded in `graphify-out/.graphify_root` to merge the new file into the existing graph. +Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. Supported URL types (auto-detected): - YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`) @@ -41,7 +41,7 @@ Supported URL types (auto-detected): Start a background watcher that monitors a folder and auto-updates the graph when files change. ```bash -$(cat graphify-out/.graphify_python) -m graphify.watch "INPUT_PATH" --debounce 3 +$(cat graphify-out/.graphify_python) -m graphify.watch INPUT_PATH --debounce 3 ``` Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: diff --git a/graphify/skills/codex/references/exports.md b/graphify/skills/codex/references/exports.md index 89ce996fb7..242ff868e0 100644 --- a/graphify/skills/codex/references/exports.md +++ b/graphify/skills/codex/references/exports.md @@ -20,27 +20,13 @@ graphify export wiki graphify export neo4j ``` -**If `--neo4j-push `** - push directly to a running Neo4j instance. Do not run these blocks through an agent tool: its shell is non-interactive and cannot read a hidden password. Show the block for the user's shell, ask them to run it in their own terminal, and wait for confirmation before continuing. Never ask them to paste the password into chat. - -POSIX shell: +**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: ```bash -NEO4J_PASSWORD="$("$(cat graphify-out/.graphify_python)" -c 'import getpass; print(getpass.getpass("Neo4j password: "))')" -export NEO4J_PASSWORD -graphify export neo4j --push bolt://localhost:7687 --user neo4j -unset NEO4J_PASSWORD -``` - -PowerShell: - -```powershell -$credential = Read-Host "Neo4j password" -AsSecureString -$env:NEO4J_PASSWORD = [System.Net.NetworkCredential]::new("", $credential).Password -graphify export neo4j --push bolt://localhost:7687 --user neo4j -Remove-Item Env:NEO4J_PASSWORD +graphify export neo4j --push bolt://localhost:7687 --user neo4j --password PASSWORD ``` -Default URI is `bolt://localhost:7687`, default user is `neo4j`. The command reads the password from `NEO4J_PASSWORD`. Uses MERGE - safe to re-run without creating duplicates. +Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. ### Step 7a - FalkorDB export (only if --falkordb or --falkordb-push flag) diff --git a/graphify/skills/codex/references/extraction-spec.md b/graphify/skills/codex/references/extraction-spec.md index 8aaa23729b..4b278b28d3 100644 --- a/graphify/skills/codex/references/extraction-spec.md +++ b/graphify/skills/codex/references/extraction-spec.md @@ -25,7 +25,7 @@ Rules: Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format `{stem}_{entity}` where stem is the full repo-relative path with the extension dropped, every segment joined with `_` (each lowercased with non-alphanumeric chars replaced by `_`) and entity is the symbol name similarly normalized. Use every directory level, not just the immediate parent. `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`. Top-level files use just the filename stem. This must match the AST extractor's ID. Never append chunk or sequence suffixes — IDs must be deterministic from the label alone. Output exactly this JSON (no other text): -{"nodes":[{"id":"src_auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} +{"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} source_file RULE: set source_file to the FILE_LIST path for that file VERBATIM (absolute, no shortening to basename, no re-relativizing, no separator change). Keeps full build and --update on one base so build_merge's replace matches instead of duplicating. ``` diff --git a/graphify/skills/codex/references/query.md b/graphify/skills/codex/references/query.md index 6dd627863c..56565eb782 100644 --- a/graphify/skills/codex/references/query.md +++ b/graphify/skills/codex/references/query.md @@ -79,12 +79,12 @@ If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal ```bash $(cat graphify-out/.graphify_python) -c " import sys, json -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph import networkx as nx from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' @@ -197,11 +197,11 @@ If the CLI is unavailable, run it inline: $(cat graphify-out/.graphify_python) -c " import json, sys import networkx as nx -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') a_term = 'NODE_A' b_term = 'NODE_B' @@ -265,11 +265,11 @@ If the CLI is unavailable, run it inline: $(cat graphify-out/.graphify_python) -c " import json, sys import networkx as nx -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') term = 'NODE_NAME' term_lower = term.lower() diff --git a/graphify/skills/codex/references/transcribe.md b/graphify/skills/codex/references/transcribe.md index bbe8b93e45..b967f83799 100644 --- a/graphify/skills/codex/references/transcribe.md +++ b/graphify/skills/codex/references/transcribe.md @@ -8,11 +8,13 @@ Skip this step entirely if `detect` returned zero `video` files. Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. -**Strategy:** If `graphify-out/.graphify_analysis.json` exists from a previous run, read its top god-node labels and write a one-sentence domain hint from them. Otherwise use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` Pass the chosen prompt to Whisper as its initial prompt. No separate API call needed. +**Strategy:** Read the god nodes from `graphify-out/.graphify_detect.json` (or the analysis file if it exists from a previous run). You are already a language model — write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. + +**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` **Step 1 - Write the Whisper prompt yourself.** -Read the top god-node labels from previous analysis when available, then compose a short domain hint sentence. With no previous analysis, use the generic fallback above. +Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: - Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` - Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` diff --git a/graphify/skills/codex/references/update.md b/graphify/skills/codex/references/update.md index 058ca8e6d2..3632fd4126 100644 --- a/graphify/skills/codex/references/update.md +++ b/graphify/skills/codex/references/update.md @@ -12,7 +12,7 @@ import sys, json from graphify.detect import detect_incremental, save_manifest from pathlib import Path -result = detect_incremental(Path(sys.argv[1])) +result = detect_incremental(Path('INPUT_PATH')) new_total = result.get('new_total', 0) print(json.dumps(result, indent=2, ensure_ascii=False)) Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") @@ -24,7 +24,7 @@ if deleted: print(f'{len(deleted)} deleted file(s) to prune.') if new_total > 0: print(f'{new_total} new/changed file(s) to re-extract.') -" "INPUT_PATH" +" ``` Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: @@ -84,7 +84,7 @@ Then: ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from pathlib import Path from graphify.build import build_merge from graphify.detect import save_manifest @@ -113,7 +113,7 @@ G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, - root=sys.argv[1], + root='INPUT_PATH', directed=IS_DIRECTED, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -150,7 +150,7 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) # is lost forever (#2015). Mirrors the library extract path # (cli._stamped_manifest_files + clear_semantic + scan_corpus). from graphify.cli import _stamped_manifest_files -_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path(sys.argv[1])) +_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) # Changed semantic files dispatched this run but NOT stamped had their chunk fail # or be omitted; clear any stale semantic_hash so they are re-queued (#1948). _sem_types = ('document', 'paper', 'image') @@ -160,9 +160,9 @@ _cleared = _dispatched - _stamped # scan_corpus = the RAW full corpus so in-root files newly excluded since last run # are dropped rather than masquerading as deletions; untouched rows preserved (#1908). _scan = {f for fl in incremental['files'].values() for f in fl} -save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) print('[graphify update] Manifest saved.') -" "INPUT_PATH" +" ``` Then run Steps 4–8 on the merged graph as normal. @@ -174,7 +174,7 @@ $(cat graphify-out/.graphify_python) -c " import json from graphify.analyze import graph_diff from graphify.build import build_from_json -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph import networkx as nx from pathlib import Path @@ -184,7 +184,7 @@ new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(e G_new = build_from_json(new_extract, directed=IS_DIRECTED) if old_data: - G_old = load_node_link_graph(old_data) + G_old = json_graph.node_link_graph(old_data, edges='links') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: diff --git a/graphify/skills/copilot/references/add-watch.md b/graphify/skills/copilot/references/add-watch.md index fcbe0cd488..77844343e1 100644 --- a/graphify/skills/copilot/references/add-watch.md +++ b/graphify/skills/copilot/references/add-watch.md @@ -13,7 +13,7 @@ from graphify.ingest import ingest from pathlib import Path try: - out = ingest(sys.argv[1], Path('./raw'), author=sys.argv[2] or None, contributor=sys.argv[3] or None) + out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') print(f'Saved to {out}') except ValueError as e: print(f'error: {e}', file=sys.stderr) @@ -21,10 +21,10 @@ except ValueError as e: except RuntimeError as e: print(f'error: {e}', file=sys.stderr) sys.exit(1) -" "URL" "AUTHOR" "CONTRIBUTOR" +" ``` -Replace `URL` with the actual URL and pass the user's `AUTHOR` or `CONTRIBUTOR` when provided; use an empty string for either omitted value. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on the scan root recorded in `graphify-out/.graphify_root` to merge the new file into the existing graph. +Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. Supported URL types (auto-detected): - YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`) @@ -41,7 +41,7 @@ Supported URL types (auto-detected): Start a background watcher that monitors a folder and auto-updates the graph when files change. ```bash -$(cat graphify-out/.graphify_python) -m graphify.watch "INPUT_PATH" --debounce 3 +$(cat graphify-out/.graphify_python) -m graphify.watch INPUT_PATH --debounce 3 ``` Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: diff --git a/graphify/skills/copilot/references/exports.md b/graphify/skills/copilot/references/exports.md index 89ce996fb7..242ff868e0 100644 --- a/graphify/skills/copilot/references/exports.md +++ b/graphify/skills/copilot/references/exports.md @@ -20,27 +20,13 @@ graphify export wiki graphify export neo4j ``` -**If `--neo4j-push `** - push directly to a running Neo4j instance. Do not run these blocks through an agent tool: its shell is non-interactive and cannot read a hidden password. Show the block for the user's shell, ask them to run it in their own terminal, and wait for confirmation before continuing. Never ask them to paste the password into chat. - -POSIX shell: +**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: ```bash -NEO4J_PASSWORD="$("$(cat graphify-out/.graphify_python)" -c 'import getpass; print(getpass.getpass("Neo4j password: "))')" -export NEO4J_PASSWORD -graphify export neo4j --push bolt://localhost:7687 --user neo4j -unset NEO4J_PASSWORD -``` - -PowerShell: - -```powershell -$credential = Read-Host "Neo4j password" -AsSecureString -$env:NEO4J_PASSWORD = [System.Net.NetworkCredential]::new("", $credential).Password -graphify export neo4j --push bolt://localhost:7687 --user neo4j -Remove-Item Env:NEO4J_PASSWORD +graphify export neo4j --push bolt://localhost:7687 --user neo4j --password PASSWORD ``` -Default URI is `bolt://localhost:7687`, default user is `neo4j`. The command reads the password from `NEO4J_PASSWORD`. Uses MERGE - safe to re-run without creating duplicates. +Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. ### Step 7a - FalkorDB export (only if --falkordb or --falkordb-push flag) diff --git a/graphify/skills/copilot/references/extraction-spec.md b/graphify/skills/copilot/references/extraction-spec.md index a5a12a575e..388df7674f 100644 --- a/graphify/skills/copilot/references/extraction-spec.md +++ b/graphify/skills/copilot/references/extraction-spec.md @@ -61,7 +61,7 @@ confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a d Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is the **full repo-relative path with the extension dropped**, every path segment kept and joined with `_` (each segment lowercased with non-alphanumeric chars replaced by `_`), and entity is the symbol name similarly normalized. Use every directory level, not just the immediate parent — this keeps same-named files in different directories distinct. Examples: `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `lib_utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`; `docs/v1/api/README.md` + `getUser` → `docs_v1_api_readme_getuser`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or only the immediate parent (e.g., `auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project built under the old immediate-parent format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. Generate the extraction JSON matching this schema exactly: -{"nodes":[{"id":"src_auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} +{"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. diff --git a/graphify/skills/copilot/references/query.md b/graphify/skills/copilot/references/query.md index 6dd627863c..56565eb782 100644 --- a/graphify/skills/copilot/references/query.md +++ b/graphify/skills/copilot/references/query.md @@ -79,12 +79,12 @@ If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal ```bash $(cat graphify-out/.graphify_python) -c " import sys, json -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph import networkx as nx from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' @@ -197,11 +197,11 @@ If the CLI is unavailable, run it inline: $(cat graphify-out/.graphify_python) -c " import json, sys import networkx as nx -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') a_term = 'NODE_A' b_term = 'NODE_B' @@ -265,11 +265,11 @@ If the CLI is unavailable, run it inline: $(cat graphify-out/.graphify_python) -c " import json, sys import networkx as nx -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') term = 'NODE_NAME' term_lower = term.lower() diff --git a/graphify/skills/copilot/references/transcribe.md b/graphify/skills/copilot/references/transcribe.md index bbe8b93e45..b967f83799 100644 --- a/graphify/skills/copilot/references/transcribe.md +++ b/graphify/skills/copilot/references/transcribe.md @@ -8,11 +8,13 @@ Skip this step entirely if `detect` returned zero `video` files. Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. -**Strategy:** If `graphify-out/.graphify_analysis.json` exists from a previous run, read its top god-node labels and write a one-sentence domain hint from them. Otherwise use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` Pass the chosen prompt to Whisper as its initial prompt. No separate API call needed. +**Strategy:** Read the god nodes from `graphify-out/.graphify_detect.json` (or the analysis file if it exists from a previous run). You are already a language model — write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. + +**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` **Step 1 - Write the Whisper prompt yourself.** -Read the top god-node labels from previous analysis when available, then compose a short domain hint sentence. With no previous analysis, use the generic fallback above. +Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: - Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` - Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` diff --git a/graphify/skills/copilot/references/update.md b/graphify/skills/copilot/references/update.md index 058ca8e6d2..3632fd4126 100644 --- a/graphify/skills/copilot/references/update.md +++ b/graphify/skills/copilot/references/update.md @@ -12,7 +12,7 @@ import sys, json from graphify.detect import detect_incremental, save_manifest from pathlib import Path -result = detect_incremental(Path(sys.argv[1])) +result = detect_incremental(Path('INPUT_PATH')) new_total = result.get('new_total', 0) print(json.dumps(result, indent=2, ensure_ascii=False)) Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") @@ -24,7 +24,7 @@ if deleted: print(f'{len(deleted)} deleted file(s) to prune.') if new_total > 0: print(f'{new_total} new/changed file(s) to re-extract.') -" "INPUT_PATH" +" ``` Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: @@ -84,7 +84,7 @@ Then: ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from pathlib import Path from graphify.build import build_merge from graphify.detect import save_manifest @@ -113,7 +113,7 @@ G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, - root=sys.argv[1], + root='INPUT_PATH', directed=IS_DIRECTED, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -150,7 +150,7 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) # is lost forever (#2015). Mirrors the library extract path # (cli._stamped_manifest_files + clear_semantic + scan_corpus). from graphify.cli import _stamped_manifest_files -_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path(sys.argv[1])) +_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) # Changed semantic files dispatched this run but NOT stamped had their chunk fail # or be omitted; clear any stale semantic_hash so they are re-queued (#1948). _sem_types = ('document', 'paper', 'image') @@ -160,9 +160,9 @@ _cleared = _dispatched - _stamped # scan_corpus = the RAW full corpus so in-root files newly excluded since last run # are dropped rather than masquerading as deletions; untouched rows preserved (#1908). _scan = {f for fl in incremental['files'].values() for f in fl} -save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) print('[graphify update] Manifest saved.') -" "INPUT_PATH" +" ``` Then run Steps 4–8 on the merged graph as normal. @@ -174,7 +174,7 @@ $(cat graphify-out/.graphify_python) -c " import json from graphify.analyze import graph_diff from graphify.build import build_from_json -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph import networkx as nx from pathlib import Path @@ -184,7 +184,7 @@ new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(e G_new = build_from_json(new_extract, directed=IS_DIRECTED) if old_data: - G_old = load_node_link_graph(old_data) + G_old = json_graph.node_link_graph(old_data, edges='links') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: diff --git a/graphify/skills/droid/references/add-watch.md b/graphify/skills/droid/references/add-watch.md index fcbe0cd488..77844343e1 100644 --- a/graphify/skills/droid/references/add-watch.md +++ b/graphify/skills/droid/references/add-watch.md @@ -13,7 +13,7 @@ from graphify.ingest import ingest from pathlib import Path try: - out = ingest(sys.argv[1], Path('./raw'), author=sys.argv[2] or None, contributor=sys.argv[3] or None) + out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') print(f'Saved to {out}') except ValueError as e: print(f'error: {e}', file=sys.stderr) @@ -21,10 +21,10 @@ except ValueError as e: except RuntimeError as e: print(f'error: {e}', file=sys.stderr) sys.exit(1) -" "URL" "AUTHOR" "CONTRIBUTOR" +" ``` -Replace `URL` with the actual URL and pass the user's `AUTHOR` or `CONTRIBUTOR` when provided; use an empty string for either omitted value. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on the scan root recorded in `graphify-out/.graphify_root` to merge the new file into the existing graph. +Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. Supported URL types (auto-detected): - YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`) @@ -41,7 +41,7 @@ Supported URL types (auto-detected): Start a background watcher that monitors a folder and auto-updates the graph when files change. ```bash -$(cat graphify-out/.graphify_python) -m graphify.watch "INPUT_PATH" --debounce 3 +$(cat graphify-out/.graphify_python) -m graphify.watch INPUT_PATH --debounce 3 ``` Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: diff --git a/graphify/skills/droid/references/exports.md b/graphify/skills/droid/references/exports.md index 89ce996fb7..242ff868e0 100644 --- a/graphify/skills/droid/references/exports.md +++ b/graphify/skills/droid/references/exports.md @@ -20,27 +20,13 @@ graphify export wiki graphify export neo4j ``` -**If `--neo4j-push `** - push directly to a running Neo4j instance. Do not run these blocks through an agent tool: its shell is non-interactive and cannot read a hidden password. Show the block for the user's shell, ask them to run it in their own terminal, and wait for confirmation before continuing. Never ask them to paste the password into chat. - -POSIX shell: +**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: ```bash -NEO4J_PASSWORD="$("$(cat graphify-out/.graphify_python)" -c 'import getpass; print(getpass.getpass("Neo4j password: "))')" -export NEO4J_PASSWORD -graphify export neo4j --push bolt://localhost:7687 --user neo4j -unset NEO4J_PASSWORD -``` - -PowerShell: - -```powershell -$credential = Read-Host "Neo4j password" -AsSecureString -$env:NEO4J_PASSWORD = [System.Net.NetworkCredential]::new("", $credential).Password -graphify export neo4j --push bolt://localhost:7687 --user neo4j -Remove-Item Env:NEO4J_PASSWORD +graphify export neo4j --push bolt://localhost:7687 --user neo4j --password PASSWORD ``` -Default URI is `bolt://localhost:7687`, default user is `neo4j`. The command reads the password from `NEO4J_PASSWORD`. Uses MERGE - safe to re-run without creating duplicates. +Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. ### Step 7a - FalkorDB export (only if --falkordb or --falkordb-push flag) diff --git a/graphify/skills/droid/references/extraction-spec.md b/graphify/skills/droid/references/extraction-spec.md index a5a12a575e..388df7674f 100644 --- a/graphify/skills/droid/references/extraction-spec.md +++ b/graphify/skills/droid/references/extraction-spec.md @@ -61,7 +61,7 @@ confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a d Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is the **full repo-relative path with the extension dropped**, every path segment kept and joined with `_` (each segment lowercased with non-alphanumeric chars replaced by `_`), and entity is the symbol name similarly normalized. Use every directory level, not just the immediate parent — this keeps same-named files in different directories distinct. Examples: `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `lib_utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`; `docs/v1/api/README.md` + `getUser` → `docs_v1_api_readme_getuser`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or only the immediate parent (e.g., `auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project built under the old immediate-parent format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. Generate the extraction JSON matching this schema exactly: -{"nodes":[{"id":"src_auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} +{"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. diff --git a/graphify/skills/droid/references/query.md b/graphify/skills/droid/references/query.md index 6dd627863c..56565eb782 100644 --- a/graphify/skills/droid/references/query.md +++ b/graphify/skills/droid/references/query.md @@ -79,12 +79,12 @@ If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal ```bash $(cat graphify-out/.graphify_python) -c " import sys, json -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph import networkx as nx from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' @@ -197,11 +197,11 @@ If the CLI is unavailable, run it inline: $(cat graphify-out/.graphify_python) -c " import json, sys import networkx as nx -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') a_term = 'NODE_A' b_term = 'NODE_B' @@ -265,11 +265,11 @@ If the CLI is unavailable, run it inline: $(cat graphify-out/.graphify_python) -c " import json, sys import networkx as nx -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') term = 'NODE_NAME' term_lower = term.lower() diff --git a/graphify/skills/droid/references/transcribe.md b/graphify/skills/droid/references/transcribe.md index bbe8b93e45..b967f83799 100644 --- a/graphify/skills/droid/references/transcribe.md +++ b/graphify/skills/droid/references/transcribe.md @@ -8,11 +8,13 @@ Skip this step entirely if `detect` returned zero `video` files. Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. -**Strategy:** If `graphify-out/.graphify_analysis.json` exists from a previous run, read its top god-node labels and write a one-sentence domain hint from them. Otherwise use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` Pass the chosen prompt to Whisper as its initial prompt. No separate API call needed. +**Strategy:** Read the god nodes from `graphify-out/.graphify_detect.json` (or the analysis file if it exists from a previous run). You are already a language model — write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. + +**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` **Step 1 - Write the Whisper prompt yourself.** -Read the top god-node labels from previous analysis when available, then compose a short domain hint sentence. With no previous analysis, use the generic fallback above. +Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: - Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` - Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` diff --git a/graphify/skills/droid/references/update.md b/graphify/skills/droid/references/update.md index 058ca8e6d2..3632fd4126 100644 --- a/graphify/skills/droid/references/update.md +++ b/graphify/skills/droid/references/update.md @@ -12,7 +12,7 @@ import sys, json from graphify.detect import detect_incremental, save_manifest from pathlib import Path -result = detect_incremental(Path(sys.argv[1])) +result = detect_incremental(Path('INPUT_PATH')) new_total = result.get('new_total', 0) print(json.dumps(result, indent=2, ensure_ascii=False)) Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") @@ -24,7 +24,7 @@ if deleted: print(f'{len(deleted)} deleted file(s) to prune.') if new_total > 0: print(f'{new_total} new/changed file(s) to re-extract.') -" "INPUT_PATH" +" ``` Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: @@ -84,7 +84,7 @@ Then: ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from pathlib import Path from graphify.build import build_merge from graphify.detect import save_manifest @@ -113,7 +113,7 @@ G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, - root=sys.argv[1], + root='INPUT_PATH', directed=IS_DIRECTED, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -150,7 +150,7 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) # is lost forever (#2015). Mirrors the library extract path # (cli._stamped_manifest_files + clear_semantic + scan_corpus). from graphify.cli import _stamped_manifest_files -_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path(sys.argv[1])) +_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) # Changed semantic files dispatched this run but NOT stamped had their chunk fail # or be omitted; clear any stale semantic_hash so they are re-queued (#1948). _sem_types = ('document', 'paper', 'image') @@ -160,9 +160,9 @@ _cleared = _dispatched - _stamped # scan_corpus = the RAW full corpus so in-root files newly excluded since last run # are dropped rather than masquerading as deletions; untouched rows preserved (#1908). _scan = {f for fl in incremental['files'].values() for f in fl} -save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) print('[graphify update] Manifest saved.') -" "INPUT_PATH" +" ``` Then run Steps 4–8 on the merged graph as normal. @@ -174,7 +174,7 @@ $(cat graphify-out/.graphify_python) -c " import json from graphify.analyze import graph_diff from graphify.build import build_from_json -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph import networkx as nx from pathlib import Path @@ -184,7 +184,7 @@ new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(e G_new = build_from_json(new_extract, directed=IS_DIRECTED) if old_data: - G_old = load_node_link_graph(old_data) + G_old = json_graph.node_link_graph(old_data, edges='links') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: diff --git a/graphify/skills/kilo/references/add-watch.md b/graphify/skills/kilo/references/add-watch.md index fcbe0cd488..77844343e1 100644 --- a/graphify/skills/kilo/references/add-watch.md +++ b/graphify/skills/kilo/references/add-watch.md @@ -13,7 +13,7 @@ from graphify.ingest import ingest from pathlib import Path try: - out = ingest(sys.argv[1], Path('./raw'), author=sys.argv[2] or None, contributor=sys.argv[3] or None) + out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') print(f'Saved to {out}') except ValueError as e: print(f'error: {e}', file=sys.stderr) @@ -21,10 +21,10 @@ except ValueError as e: except RuntimeError as e: print(f'error: {e}', file=sys.stderr) sys.exit(1) -" "URL" "AUTHOR" "CONTRIBUTOR" +" ``` -Replace `URL` with the actual URL and pass the user's `AUTHOR` or `CONTRIBUTOR` when provided; use an empty string for either omitted value. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on the scan root recorded in `graphify-out/.graphify_root` to merge the new file into the existing graph. +Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. Supported URL types (auto-detected): - YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`) @@ -41,7 +41,7 @@ Supported URL types (auto-detected): Start a background watcher that monitors a folder and auto-updates the graph when files change. ```bash -$(cat graphify-out/.graphify_python) -m graphify.watch "INPUT_PATH" --debounce 3 +$(cat graphify-out/.graphify_python) -m graphify.watch INPUT_PATH --debounce 3 ``` Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: diff --git a/graphify/skills/kilo/references/exports.md b/graphify/skills/kilo/references/exports.md index 89ce996fb7..242ff868e0 100644 --- a/graphify/skills/kilo/references/exports.md +++ b/graphify/skills/kilo/references/exports.md @@ -20,27 +20,13 @@ graphify export wiki graphify export neo4j ``` -**If `--neo4j-push `** - push directly to a running Neo4j instance. Do not run these blocks through an agent tool: its shell is non-interactive and cannot read a hidden password. Show the block for the user's shell, ask them to run it in their own terminal, and wait for confirmation before continuing. Never ask them to paste the password into chat. - -POSIX shell: +**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: ```bash -NEO4J_PASSWORD="$("$(cat graphify-out/.graphify_python)" -c 'import getpass; print(getpass.getpass("Neo4j password: "))')" -export NEO4J_PASSWORD -graphify export neo4j --push bolt://localhost:7687 --user neo4j -unset NEO4J_PASSWORD -``` - -PowerShell: - -```powershell -$credential = Read-Host "Neo4j password" -AsSecureString -$env:NEO4J_PASSWORD = [System.Net.NetworkCredential]::new("", $credential).Password -graphify export neo4j --push bolt://localhost:7687 --user neo4j -Remove-Item Env:NEO4J_PASSWORD +graphify export neo4j --push bolt://localhost:7687 --user neo4j --password PASSWORD ``` -Default URI is `bolt://localhost:7687`, default user is `neo4j`. The command reads the password from `NEO4J_PASSWORD`. Uses MERGE - safe to re-run without creating duplicates. +Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. ### Step 7a - FalkorDB export (only if --falkordb or --falkordb-push flag) diff --git a/graphify/skills/kilo/references/extraction-spec.md b/graphify/skills/kilo/references/extraction-spec.md index a5a12a575e..388df7674f 100644 --- a/graphify/skills/kilo/references/extraction-spec.md +++ b/graphify/skills/kilo/references/extraction-spec.md @@ -61,7 +61,7 @@ confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a d Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is the **full repo-relative path with the extension dropped**, every path segment kept and joined with `_` (each segment lowercased with non-alphanumeric chars replaced by `_`), and entity is the symbol name similarly normalized. Use every directory level, not just the immediate parent — this keeps same-named files in different directories distinct. Examples: `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `lib_utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`; `docs/v1/api/README.md` + `getUser` → `docs_v1_api_readme_getuser`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or only the immediate parent (e.g., `auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project built under the old immediate-parent format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. Generate the extraction JSON matching this schema exactly: -{"nodes":[{"id":"src_auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} +{"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. diff --git a/graphify/skills/kilo/references/query.md b/graphify/skills/kilo/references/query.md index 6dd627863c..56565eb782 100644 --- a/graphify/skills/kilo/references/query.md +++ b/graphify/skills/kilo/references/query.md @@ -79,12 +79,12 @@ If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal ```bash $(cat graphify-out/.graphify_python) -c " import sys, json -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph import networkx as nx from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' @@ -197,11 +197,11 @@ If the CLI is unavailable, run it inline: $(cat graphify-out/.graphify_python) -c " import json, sys import networkx as nx -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') a_term = 'NODE_A' b_term = 'NODE_B' @@ -265,11 +265,11 @@ If the CLI is unavailable, run it inline: $(cat graphify-out/.graphify_python) -c " import json, sys import networkx as nx -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') term = 'NODE_NAME' term_lower = term.lower() diff --git a/graphify/skills/kilo/references/transcribe.md b/graphify/skills/kilo/references/transcribe.md index bbe8b93e45..b967f83799 100644 --- a/graphify/skills/kilo/references/transcribe.md +++ b/graphify/skills/kilo/references/transcribe.md @@ -8,11 +8,13 @@ Skip this step entirely if `detect` returned zero `video` files. Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. -**Strategy:** If `graphify-out/.graphify_analysis.json` exists from a previous run, read its top god-node labels and write a one-sentence domain hint from them. Otherwise use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` Pass the chosen prompt to Whisper as its initial prompt. No separate API call needed. +**Strategy:** Read the god nodes from `graphify-out/.graphify_detect.json` (or the analysis file if it exists from a previous run). You are already a language model — write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. + +**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` **Step 1 - Write the Whisper prompt yourself.** -Read the top god-node labels from previous analysis when available, then compose a short domain hint sentence. With no previous analysis, use the generic fallback above. +Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: - Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` - Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` diff --git a/graphify/skills/kilo/references/update.md b/graphify/skills/kilo/references/update.md index 058ca8e6d2..3632fd4126 100644 --- a/graphify/skills/kilo/references/update.md +++ b/graphify/skills/kilo/references/update.md @@ -12,7 +12,7 @@ import sys, json from graphify.detect import detect_incremental, save_manifest from pathlib import Path -result = detect_incremental(Path(sys.argv[1])) +result = detect_incremental(Path('INPUT_PATH')) new_total = result.get('new_total', 0) print(json.dumps(result, indent=2, ensure_ascii=False)) Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") @@ -24,7 +24,7 @@ if deleted: print(f'{len(deleted)} deleted file(s) to prune.') if new_total > 0: print(f'{new_total} new/changed file(s) to re-extract.') -" "INPUT_PATH" +" ``` Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: @@ -84,7 +84,7 @@ Then: ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from pathlib import Path from graphify.build import build_merge from graphify.detect import save_manifest @@ -113,7 +113,7 @@ G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, - root=sys.argv[1], + root='INPUT_PATH', directed=IS_DIRECTED, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -150,7 +150,7 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) # is lost forever (#2015). Mirrors the library extract path # (cli._stamped_manifest_files + clear_semantic + scan_corpus). from graphify.cli import _stamped_manifest_files -_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path(sys.argv[1])) +_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) # Changed semantic files dispatched this run but NOT stamped had their chunk fail # or be omitted; clear any stale semantic_hash so they are re-queued (#1948). _sem_types = ('document', 'paper', 'image') @@ -160,9 +160,9 @@ _cleared = _dispatched - _stamped # scan_corpus = the RAW full corpus so in-root files newly excluded since last run # are dropped rather than masquerading as deletions; untouched rows preserved (#1908). _scan = {f for fl in incremental['files'].values() for f in fl} -save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) print('[graphify update] Manifest saved.') -" "INPUT_PATH" +" ``` Then run Steps 4–8 on the merged graph as normal. @@ -174,7 +174,7 @@ $(cat graphify-out/.graphify_python) -c " import json from graphify.analyze import graph_diff from graphify.build import build_from_json -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph import networkx as nx from pathlib import Path @@ -184,7 +184,7 @@ new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(e G_new = build_from_json(new_extract, directed=IS_DIRECTED) if old_data: - G_old = load_node_link_graph(old_data) + G_old = json_graph.node_link_graph(old_data, edges='links') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: diff --git a/graphify/skills/kiro/references/add-watch.md b/graphify/skills/kiro/references/add-watch.md index fcbe0cd488..77844343e1 100644 --- a/graphify/skills/kiro/references/add-watch.md +++ b/graphify/skills/kiro/references/add-watch.md @@ -13,7 +13,7 @@ from graphify.ingest import ingest from pathlib import Path try: - out = ingest(sys.argv[1], Path('./raw'), author=sys.argv[2] or None, contributor=sys.argv[3] or None) + out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') print(f'Saved to {out}') except ValueError as e: print(f'error: {e}', file=sys.stderr) @@ -21,10 +21,10 @@ except ValueError as e: except RuntimeError as e: print(f'error: {e}', file=sys.stderr) sys.exit(1) -" "URL" "AUTHOR" "CONTRIBUTOR" +" ``` -Replace `URL` with the actual URL and pass the user's `AUTHOR` or `CONTRIBUTOR` when provided; use an empty string for either omitted value. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on the scan root recorded in `graphify-out/.graphify_root` to merge the new file into the existing graph. +Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. Supported URL types (auto-detected): - YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`) @@ -41,7 +41,7 @@ Supported URL types (auto-detected): Start a background watcher that monitors a folder and auto-updates the graph when files change. ```bash -$(cat graphify-out/.graphify_python) -m graphify.watch "INPUT_PATH" --debounce 3 +$(cat graphify-out/.graphify_python) -m graphify.watch INPUT_PATH --debounce 3 ``` Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: diff --git a/graphify/skills/kiro/references/exports.md b/graphify/skills/kiro/references/exports.md index 89ce996fb7..242ff868e0 100644 --- a/graphify/skills/kiro/references/exports.md +++ b/graphify/skills/kiro/references/exports.md @@ -20,27 +20,13 @@ graphify export wiki graphify export neo4j ``` -**If `--neo4j-push `** - push directly to a running Neo4j instance. Do not run these blocks through an agent tool: its shell is non-interactive and cannot read a hidden password. Show the block for the user's shell, ask them to run it in their own terminal, and wait for confirmation before continuing. Never ask them to paste the password into chat. - -POSIX shell: +**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: ```bash -NEO4J_PASSWORD="$("$(cat graphify-out/.graphify_python)" -c 'import getpass; print(getpass.getpass("Neo4j password: "))')" -export NEO4J_PASSWORD -graphify export neo4j --push bolt://localhost:7687 --user neo4j -unset NEO4J_PASSWORD -``` - -PowerShell: - -```powershell -$credential = Read-Host "Neo4j password" -AsSecureString -$env:NEO4J_PASSWORD = [System.Net.NetworkCredential]::new("", $credential).Password -graphify export neo4j --push bolt://localhost:7687 --user neo4j -Remove-Item Env:NEO4J_PASSWORD +graphify export neo4j --push bolt://localhost:7687 --user neo4j --password PASSWORD ``` -Default URI is `bolt://localhost:7687`, default user is `neo4j`. The command reads the password from `NEO4J_PASSWORD`. Uses MERGE - safe to re-run without creating duplicates. +Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. ### Step 7a - FalkorDB export (only if --falkordb or --falkordb-push flag) diff --git a/graphify/skills/kiro/references/extraction-spec.md b/graphify/skills/kiro/references/extraction-spec.md index 8aaa23729b..4b278b28d3 100644 --- a/graphify/skills/kiro/references/extraction-spec.md +++ b/graphify/skills/kiro/references/extraction-spec.md @@ -25,7 +25,7 @@ Rules: Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format `{stem}_{entity}` where stem is the full repo-relative path with the extension dropped, every segment joined with `_` (each lowercased with non-alphanumeric chars replaced by `_`) and entity is the symbol name similarly normalized. Use every directory level, not just the immediate parent. `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`. Top-level files use just the filename stem. This must match the AST extractor's ID. Never append chunk or sequence suffixes — IDs must be deterministic from the label alone. Output exactly this JSON (no other text): -{"nodes":[{"id":"src_auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} +{"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} source_file RULE: set source_file to the FILE_LIST path for that file VERBATIM (absolute, no shortening to basename, no re-relativizing, no separator change). Keeps full build and --update on one base so build_merge's replace matches instead of duplicating. ``` diff --git a/graphify/skills/kiro/references/query.md b/graphify/skills/kiro/references/query.md index 6dd627863c..56565eb782 100644 --- a/graphify/skills/kiro/references/query.md +++ b/graphify/skills/kiro/references/query.md @@ -79,12 +79,12 @@ If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal ```bash $(cat graphify-out/.graphify_python) -c " import sys, json -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph import networkx as nx from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' @@ -197,11 +197,11 @@ If the CLI is unavailable, run it inline: $(cat graphify-out/.graphify_python) -c " import json, sys import networkx as nx -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') a_term = 'NODE_A' b_term = 'NODE_B' @@ -265,11 +265,11 @@ If the CLI is unavailable, run it inline: $(cat graphify-out/.graphify_python) -c " import json, sys import networkx as nx -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') term = 'NODE_NAME' term_lower = term.lower() diff --git a/graphify/skills/kiro/references/transcribe.md b/graphify/skills/kiro/references/transcribe.md index bbe8b93e45..b967f83799 100644 --- a/graphify/skills/kiro/references/transcribe.md +++ b/graphify/skills/kiro/references/transcribe.md @@ -8,11 +8,13 @@ Skip this step entirely if `detect` returned zero `video` files. Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. -**Strategy:** If `graphify-out/.graphify_analysis.json` exists from a previous run, read its top god-node labels and write a one-sentence domain hint from them. Otherwise use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` Pass the chosen prompt to Whisper as its initial prompt. No separate API call needed. +**Strategy:** Read the god nodes from `graphify-out/.graphify_detect.json` (or the analysis file if it exists from a previous run). You are already a language model — write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. + +**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` **Step 1 - Write the Whisper prompt yourself.** -Read the top god-node labels from previous analysis when available, then compose a short domain hint sentence. With no previous analysis, use the generic fallback above. +Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: - Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` - Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` diff --git a/graphify/skills/kiro/references/update.md b/graphify/skills/kiro/references/update.md index 058ca8e6d2..3632fd4126 100644 --- a/graphify/skills/kiro/references/update.md +++ b/graphify/skills/kiro/references/update.md @@ -12,7 +12,7 @@ import sys, json from graphify.detect import detect_incremental, save_manifest from pathlib import Path -result = detect_incremental(Path(sys.argv[1])) +result = detect_incremental(Path('INPUT_PATH')) new_total = result.get('new_total', 0) print(json.dumps(result, indent=2, ensure_ascii=False)) Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") @@ -24,7 +24,7 @@ if deleted: print(f'{len(deleted)} deleted file(s) to prune.') if new_total > 0: print(f'{new_total} new/changed file(s) to re-extract.') -" "INPUT_PATH" +" ``` Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: @@ -84,7 +84,7 @@ Then: ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from pathlib import Path from graphify.build import build_merge from graphify.detect import save_manifest @@ -113,7 +113,7 @@ G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, - root=sys.argv[1], + root='INPUT_PATH', directed=IS_DIRECTED, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -150,7 +150,7 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) # is lost forever (#2015). Mirrors the library extract path # (cli._stamped_manifest_files + clear_semantic + scan_corpus). from graphify.cli import _stamped_manifest_files -_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path(sys.argv[1])) +_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) # Changed semantic files dispatched this run but NOT stamped had their chunk fail # or be omitted; clear any stale semantic_hash so they are re-queued (#1948). _sem_types = ('document', 'paper', 'image') @@ -160,9 +160,9 @@ _cleared = _dispatched - _stamped # scan_corpus = the RAW full corpus so in-root files newly excluded since last run # are dropped rather than masquerading as deletions; untouched rows preserved (#1908). _scan = {f for fl in incremental['files'].values() for f in fl} -save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) print('[graphify update] Manifest saved.') -" "INPUT_PATH" +" ``` Then run Steps 4–8 on the merged graph as normal. @@ -174,7 +174,7 @@ $(cat graphify-out/.graphify_python) -c " import json from graphify.analyze import graph_diff from graphify.build import build_from_json -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph import networkx as nx from pathlib import Path @@ -184,7 +184,7 @@ new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(e G_new = build_from_json(new_extract, directed=IS_DIRECTED) if old_data: - G_old = load_node_link_graph(old_data) + G_old = json_graph.node_link_graph(old_data, edges='links') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: diff --git a/graphify/skills/opencode/references/add-watch.md b/graphify/skills/opencode/references/add-watch.md index fcbe0cd488..77844343e1 100644 --- a/graphify/skills/opencode/references/add-watch.md +++ b/graphify/skills/opencode/references/add-watch.md @@ -13,7 +13,7 @@ from graphify.ingest import ingest from pathlib import Path try: - out = ingest(sys.argv[1], Path('./raw'), author=sys.argv[2] or None, contributor=sys.argv[3] or None) + out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') print(f'Saved to {out}') except ValueError as e: print(f'error: {e}', file=sys.stderr) @@ -21,10 +21,10 @@ except ValueError as e: except RuntimeError as e: print(f'error: {e}', file=sys.stderr) sys.exit(1) -" "URL" "AUTHOR" "CONTRIBUTOR" +" ``` -Replace `URL` with the actual URL and pass the user's `AUTHOR` or `CONTRIBUTOR` when provided; use an empty string for either omitted value. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on the scan root recorded in `graphify-out/.graphify_root` to merge the new file into the existing graph. +Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. Supported URL types (auto-detected): - YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`) @@ -41,7 +41,7 @@ Supported URL types (auto-detected): Start a background watcher that monitors a folder and auto-updates the graph when files change. ```bash -$(cat graphify-out/.graphify_python) -m graphify.watch "INPUT_PATH" --debounce 3 +$(cat graphify-out/.graphify_python) -m graphify.watch INPUT_PATH --debounce 3 ``` Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: diff --git a/graphify/skills/opencode/references/exports.md b/graphify/skills/opencode/references/exports.md index 89ce996fb7..242ff868e0 100644 --- a/graphify/skills/opencode/references/exports.md +++ b/graphify/skills/opencode/references/exports.md @@ -20,27 +20,13 @@ graphify export wiki graphify export neo4j ``` -**If `--neo4j-push `** - push directly to a running Neo4j instance. Do not run these blocks through an agent tool: its shell is non-interactive and cannot read a hidden password. Show the block for the user's shell, ask them to run it in their own terminal, and wait for confirmation before continuing. Never ask them to paste the password into chat. - -POSIX shell: +**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: ```bash -NEO4J_PASSWORD="$("$(cat graphify-out/.graphify_python)" -c 'import getpass; print(getpass.getpass("Neo4j password: "))')" -export NEO4J_PASSWORD -graphify export neo4j --push bolt://localhost:7687 --user neo4j -unset NEO4J_PASSWORD -``` - -PowerShell: - -```powershell -$credential = Read-Host "Neo4j password" -AsSecureString -$env:NEO4J_PASSWORD = [System.Net.NetworkCredential]::new("", $credential).Password -graphify export neo4j --push bolt://localhost:7687 --user neo4j -Remove-Item Env:NEO4J_PASSWORD +graphify export neo4j --push bolt://localhost:7687 --user neo4j --password PASSWORD ``` -Default URI is `bolt://localhost:7687`, default user is `neo4j`. The command reads the password from `NEO4J_PASSWORD`. Uses MERGE - safe to re-run without creating duplicates. +Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. ### Step 7a - FalkorDB export (only if --falkordb or --falkordb-push flag) diff --git a/graphify/skills/opencode/references/extraction-spec.md b/graphify/skills/opencode/references/extraction-spec.md index a5a12a575e..388df7674f 100644 --- a/graphify/skills/opencode/references/extraction-spec.md +++ b/graphify/skills/opencode/references/extraction-spec.md @@ -61,7 +61,7 @@ confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a d Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is the **full repo-relative path with the extension dropped**, every path segment kept and joined with `_` (each segment lowercased with non-alphanumeric chars replaced by `_`), and entity is the symbol name similarly normalized. Use every directory level, not just the immediate parent — this keeps same-named files in different directories distinct. Examples: `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `lib_utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`; `docs/v1/api/README.md` + `getUser` → `docs_v1_api_readme_getuser`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or only the immediate parent (e.g., `auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project built under the old immediate-parent format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. Generate the extraction JSON matching this schema exactly: -{"nodes":[{"id":"src_auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} +{"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. diff --git a/graphify/skills/opencode/references/query.md b/graphify/skills/opencode/references/query.md index 6dd627863c..56565eb782 100644 --- a/graphify/skills/opencode/references/query.md +++ b/graphify/skills/opencode/references/query.md @@ -79,12 +79,12 @@ If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal ```bash $(cat graphify-out/.graphify_python) -c " import sys, json -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph import networkx as nx from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' @@ -197,11 +197,11 @@ If the CLI is unavailable, run it inline: $(cat graphify-out/.graphify_python) -c " import json, sys import networkx as nx -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') a_term = 'NODE_A' b_term = 'NODE_B' @@ -265,11 +265,11 @@ If the CLI is unavailable, run it inline: $(cat graphify-out/.graphify_python) -c " import json, sys import networkx as nx -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') term = 'NODE_NAME' term_lower = term.lower() diff --git a/graphify/skills/opencode/references/transcribe.md b/graphify/skills/opencode/references/transcribe.md index bbe8b93e45..b967f83799 100644 --- a/graphify/skills/opencode/references/transcribe.md +++ b/graphify/skills/opencode/references/transcribe.md @@ -8,11 +8,13 @@ Skip this step entirely if `detect` returned zero `video` files. Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. -**Strategy:** If `graphify-out/.graphify_analysis.json` exists from a previous run, read its top god-node labels and write a one-sentence domain hint from them. Otherwise use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` Pass the chosen prompt to Whisper as its initial prompt. No separate API call needed. +**Strategy:** Read the god nodes from `graphify-out/.graphify_detect.json` (or the analysis file if it exists from a previous run). You are already a language model — write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. + +**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` **Step 1 - Write the Whisper prompt yourself.** -Read the top god-node labels from previous analysis when available, then compose a short domain hint sentence. With no previous analysis, use the generic fallback above. +Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: - Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` - Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` diff --git a/graphify/skills/opencode/references/update.md b/graphify/skills/opencode/references/update.md index 058ca8e6d2..3632fd4126 100644 --- a/graphify/skills/opencode/references/update.md +++ b/graphify/skills/opencode/references/update.md @@ -12,7 +12,7 @@ import sys, json from graphify.detect import detect_incremental, save_manifest from pathlib import Path -result = detect_incremental(Path(sys.argv[1])) +result = detect_incremental(Path('INPUT_PATH')) new_total = result.get('new_total', 0) print(json.dumps(result, indent=2, ensure_ascii=False)) Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") @@ -24,7 +24,7 @@ if deleted: print(f'{len(deleted)} deleted file(s) to prune.') if new_total > 0: print(f'{new_total} new/changed file(s) to re-extract.') -" "INPUT_PATH" +" ``` Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: @@ -84,7 +84,7 @@ Then: ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from pathlib import Path from graphify.build import build_merge from graphify.detect import save_manifest @@ -113,7 +113,7 @@ G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, - root=sys.argv[1], + root='INPUT_PATH', directed=IS_DIRECTED, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -150,7 +150,7 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) # is lost forever (#2015). Mirrors the library extract path # (cli._stamped_manifest_files + clear_semantic + scan_corpus). from graphify.cli import _stamped_manifest_files -_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path(sys.argv[1])) +_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) # Changed semantic files dispatched this run but NOT stamped had their chunk fail # or be omitted; clear any stale semantic_hash so they are re-queued (#1948). _sem_types = ('document', 'paper', 'image') @@ -160,9 +160,9 @@ _cleared = _dispatched - _stamped # scan_corpus = the RAW full corpus so in-root files newly excluded since last run # are dropped rather than masquerading as deletions; untouched rows preserved (#1908). _scan = {f for fl in incremental['files'].values() for f in fl} -save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) print('[graphify update] Manifest saved.') -" "INPUT_PATH" +" ``` Then run Steps 4–8 on the merged graph as normal. @@ -174,7 +174,7 @@ $(cat graphify-out/.graphify_python) -c " import json from graphify.analyze import graph_diff from graphify.build import build_from_json -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph import networkx as nx from pathlib import Path @@ -184,7 +184,7 @@ new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(e G_new = build_from_json(new_extract, directed=IS_DIRECTED) if old_data: - G_old = load_node_link_graph(old_data) + G_old = json_graph.node_link_graph(old_data, edges='links') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: diff --git a/graphify/skills/pi/references/add-watch.md b/graphify/skills/pi/references/add-watch.md index fcbe0cd488..77844343e1 100644 --- a/graphify/skills/pi/references/add-watch.md +++ b/graphify/skills/pi/references/add-watch.md @@ -13,7 +13,7 @@ from graphify.ingest import ingest from pathlib import Path try: - out = ingest(sys.argv[1], Path('./raw'), author=sys.argv[2] or None, contributor=sys.argv[3] or None) + out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') print(f'Saved to {out}') except ValueError as e: print(f'error: {e}', file=sys.stderr) @@ -21,10 +21,10 @@ except ValueError as e: except RuntimeError as e: print(f'error: {e}', file=sys.stderr) sys.exit(1) -" "URL" "AUTHOR" "CONTRIBUTOR" +" ``` -Replace `URL` with the actual URL and pass the user's `AUTHOR` or `CONTRIBUTOR` when provided; use an empty string for either omitted value. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on the scan root recorded in `graphify-out/.graphify_root` to merge the new file into the existing graph. +Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. Supported URL types (auto-detected): - YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`) @@ -41,7 +41,7 @@ Supported URL types (auto-detected): Start a background watcher that monitors a folder and auto-updates the graph when files change. ```bash -$(cat graphify-out/.graphify_python) -m graphify.watch "INPUT_PATH" --debounce 3 +$(cat graphify-out/.graphify_python) -m graphify.watch INPUT_PATH --debounce 3 ``` Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: diff --git a/graphify/skills/pi/references/exports.md b/graphify/skills/pi/references/exports.md index 89ce996fb7..242ff868e0 100644 --- a/graphify/skills/pi/references/exports.md +++ b/graphify/skills/pi/references/exports.md @@ -20,27 +20,13 @@ graphify export wiki graphify export neo4j ``` -**If `--neo4j-push `** - push directly to a running Neo4j instance. Do not run these blocks through an agent tool: its shell is non-interactive and cannot read a hidden password. Show the block for the user's shell, ask them to run it in their own terminal, and wait for confirmation before continuing. Never ask them to paste the password into chat. - -POSIX shell: +**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: ```bash -NEO4J_PASSWORD="$("$(cat graphify-out/.graphify_python)" -c 'import getpass; print(getpass.getpass("Neo4j password: "))')" -export NEO4J_PASSWORD -graphify export neo4j --push bolt://localhost:7687 --user neo4j -unset NEO4J_PASSWORD -``` - -PowerShell: - -```powershell -$credential = Read-Host "Neo4j password" -AsSecureString -$env:NEO4J_PASSWORD = [System.Net.NetworkCredential]::new("", $credential).Password -graphify export neo4j --push bolt://localhost:7687 --user neo4j -Remove-Item Env:NEO4J_PASSWORD +graphify export neo4j --push bolt://localhost:7687 --user neo4j --password PASSWORD ``` -Default URI is `bolt://localhost:7687`, default user is `neo4j`. The command reads the password from `NEO4J_PASSWORD`. Uses MERGE - safe to re-run without creating duplicates. +Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. ### Step 7a - FalkorDB export (only if --falkordb or --falkordb-push flag) diff --git a/graphify/skills/pi/references/extraction-spec.md b/graphify/skills/pi/references/extraction-spec.md index 8aaa23729b..4b278b28d3 100644 --- a/graphify/skills/pi/references/extraction-spec.md +++ b/graphify/skills/pi/references/extraction-spec.md @@ -25,7 +25,7 @@ Rules: Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format `{stem}_{entity}` where stem is the full repo-relative path with the extension dropped, every segment joined with `_` (each lowercased with non-alphanumeric chars replaced by `_`) and entity is the symbol name similarly normalized. Use every directory level, not just the immediate parent. `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`. Top-level files use just the filename stem. This must match the AST extractor's ID. Never append chunk or sequence suffixes — IDs must be deterministic from the label alone. Output exactly this JSON (no other text): -{"nodes":[{"id":"src_auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} +{"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} source_file RULE: set source_file to the FILE_LIST path for that file VERBATIM (absolute, no shortening to basename, no re-relativizing, no separator change). Keeps full build and --update on one base so build_merge's replace matches instead of duplicating. ``` diff --git a/graphify/skills/pi/references/query.md b/graphify/skills/pi/references/query.md index 6dd627863c..56565eb782 100644 --- a/graphify/skills/pi/references/query.md +++ b/graphify/skills/pi/references/query.md @@ -79,12 +79,12 @@ If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal ```bash $(cat graphify-out/.graphify_python) -c " import sys, json -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph import networkx as nx from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' @@ -197,11 +197,11 @@ If the CLI is unavailable, run it inline: $(cat graphify-out/.graphify_python) -c " import json, sys import networkx as nx -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') a_term = 'NODE_A' b_term = 'NODE_B' @@ -265,11 +265,11 @@ If the CLI is unavailable, run it inline: $(cat graphify-out/.graphify_python) -c " import json, sys import networkx as nx -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') term = 'NODE_NAME' term_lower = term.lower() diff --git a/graphify/skills/pi/references/transcribe.md b/graphify/skills/pi/references/transcribe.md index bbe8b93e45..b967f83799 100644 --- a/graphify/skills/pi/references/transcribe.md +++ b/graphify/skills/pi/references/transcribe.md @@ -8,11 +8,13 @@ Skip this step entirely if `detect` returned zero `video` files. Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. -**Strategy:** If `graphify-out/.graphify_analysis.json` exists from a previous run, read its top god-node labels and write a one-sentence domain hint from them. Otherwise use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` Pass the chosen prompt to Whisper as its initial prompt. No separate API call needed. +**Strategy:** Read the god nodes from `graphify-out/.graphify_detect.json` (or the analysis file if it exists from a previous run). You are already a language model — write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. + +**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` **Step 1 - Write the Whisper prompt yourself.** -Read the top god-node labels from previous analysis when available, then compose a short domain hint sentence. With no previous analysis, use the generic fallback above. +Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: - Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` - Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` diff --git a/graphify/skills/pi/references/update.md b/graphify/skills/pi/references/update.md index 058ca8e6d2..3632fd4126 100644 --- a/graphify/skills/pi/references/update.md +++ b/graphify/skills/pi/references/update.md @@ -12,7 +12,7 @@ import sys, json from graphify.detect import detect_incremental, save_manifest from pathlib import Path -result = detect_incremental(Path(sys.argv[1])) +result = detect_incremental(Path('INPUT_PATH')) new_total = result.get('new_total', 0) print(json.dumps(result, indent=2, ensure_ascii=False)) Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") @@ -24,7 +24,7 @@ if deleted: print(f'{len(deleted)} deleted file(s) to prune.') if new_total > 0: print(f'{new_total} new/changed file(s) to re-extract.') -" "INPUT_PATH" +" ``` Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: @@ -84,7 +84,7 @@ Then: ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from pathlib import Path from graphify.build import build_merge from graphify.detect import save_manifest @@ -113,7 +113,7 @@ G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, - root=sys.argv[1], + root='INPUT_PATH', directed=IS_DIRECTED, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -150,7 +150,7 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) # is lost forever (#2015). Mirrors the library extract path # (cli._stamped_manifest_files + clear_semantic + scan_corpus). from graphify.cli import _stamped_manifest_files -_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path(sys.argv[1])) +_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) # Changed semantic files dispatched this run but NOT stamped had their chunk fail # or be omitted; clear any stale semantic_hash so they are re-queued (#1948). _sem_types = ('document', 'paper', 'image') @@ -160,9 +160,9 @@ _cleared = _dispatched - _stamped # scan_corpus = the RAW full corpus so in-root files newly excluded since last run # are dropped rather than masquerading as deletions; untouched rows preserved (#1908). _scan = {f for fl in incremental['files'].values() for f in fl} -save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) print('[graphify update] Manifest saved.') -" "INPUT_PATH" +" ``` Then run Steps 4–8 on the merged graph as normal. @@ -174,7 +174,7 @@ $(cat graphify-out/.graphify_python) -c " import json from graphify.analyze import graph_diff from graphify.build import build_from_json -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph import networkx as nx from pathlib import Path @@ -184,7 +184,7 @@ new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(e G_new = build_from_json(new_extract, directed=IS_DIRECTED) if old_data: - G_old = load_node_link_graph(old_data) + G_old = json_graph.node_link_graph(old_data, edges='links') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: diff --git a/graphify/skills/trae/references/add-watch.md b/graphify/skills/trae/references/add-watch.md index fcbe0cd488..77844343e1 100644 --- a/graphify/skills/trae/references/add-watch.md +++ b/graphify/skills/trae/references/add-watch.md @@ -13,7 +13,7 @@ from graphify.ingest import ingest from pathlib import Path try: - out = ingest(sys.argv[1], Path('./raw'), author=sys.argv[2] or None, contributor=sys.argv[3] or None) + out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') print(f'Saved to {out}') except ValueError as e: print(f'error: {e}', file=sys.stderr) @@ -21,10 +21,10 @@ except ValueError as e: except RuntimeError as e: print(f'error: {e}', file=sys.stderr) sys.exit(1) -" "URL" "AUTHOR" "CONTRIBUTOR" +" ``` -Replace `URL` with the actual URL and pass the user's `AUTHOR` or `CONTRIBUTOR` when provided; use an empty string for either omitted value. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on the scan root recorded in `graphify-out/.graphify_root` to merge the new file into the existing graph. +Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. Supported URL types (auto-detected): - YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`) @@ -41,7 +41,7 @@ Supported URL types (auto-detected): Start a background watcher that monitors a folder and auto-updates the graph when files change. ```bash -$(cat graphify-out/.graphify_python) -m graphify.watch "INPUT_PATH" --debounce 3 +$(cat graphify-out/.graphify_python) -m graphify.watch INPUT_PATH --debounce 3 ``` Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: diff --git a/graphify/skills/trae/references/exports.md b/graphify/skills/trae/references/exports.md index 89ce996fb7..242ff868e0 100644 --- a/graphify/skills/trae/references/exports.md +++ b/graphify/skills/trae/references/exports.md @@ -20,27 +20,13 @@ graphify export wiki graphify export neo4j ``` -**If `--neo4j-push `** - push directly to a running Neo4j instance. Do not run these blocks through an agent tool: its shell is non-interactive and cannot read a hidden password. Show the block for the user's shell, ask them to run it in their own terminal, and wait for confirmation before continuing. Never ask them to paste the password into chat. - -POSIX shell: +**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: ```bash -NEO4J_PASSWORD="$("$(cat graphify-out/.graphify_python)" -c 'import getpass; print(getpass.getpass("Neo4j password: "))')" -export NEO4J_PASSWORD -graphify export neo4j --push bolt://localhost:7687 --user neo4j -unset NEO4J_PASSWORD -``` - -PowerShell: - -```powershell -$credential = Read-Host "Neo4j password" -AsSecureString -$env:NEO4J_PASSWORD = [System.Net.NetworkCredential]::new("", $credential).Password -graphify export neo4j --push bolt://localhost:7687 --user neo4j -Remove-Item Env:NEO4J_PASSWORD +graphify export neo4j --push bolt://localhost:7687 --user neo4j --password PASSWORD ``` -Default URI is `bolt://localhost:7687`, default user is `neo4j`. The command reads the password from `NEO4J_PASSWORD`. Uses MERGE - safe to re-run without creating duplicates. +Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. ### Step 7a - FalkorDB export (only if --falkordb or --falkordb-push flag) diff --git a/graphify/skills/trae/references/extraction-spec.md b/graphify/skills/trae/references/extraction-spec.md index a5a12a575e..388df7674f 100644 --- a/graphify/skills/trae/references/extraction-spec.md +++ b/graphify/skills/trae/references/extraction-spec.md @@ -61,7 +61,7 @@ confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a d Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is the **full repo-relative path with the extension dropped**, every path segment kept and joined with `_` (each segment lowercased with non-alphanumeric chars replaced by `_`), and entity is the symbol name similarly normalized. Use every directory level, not just the immediate parent — this keeps same-named files in different directories distinct. Examples: `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `lib_utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`; `docs/v1/api/README.md` + `getUser` → `docs_v1_api_readme_getuser`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or only the immediate parent (e.g., `auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project built under the old immediate-parent format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. Generate the extraction JSON matching this schema exactly: -{"nodes":[{"id":"src_auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} +{"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. diff --git a/graphify/skills/trae/references/query.md b/graphify/skills/trae/references/query.md index 6dd627863c..56565eb782 100644 --- a/graphify/skills/trae/references/query.md +++ b/graphify/skills/trae/references/query.md @@ -79,12 +79,12 @@ If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal ```bash $(cat graphify-out/.graphify_python) -c " import sys, json -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph import networkx as nx from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' @@ -197,11 +197,11 @@ If the CLI is unavailable, run it inline: $(cat graphify-out/.graphify_python) -c " import json, sys import networkx as nx -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') a_term = 'NODE_A' b_term = 'NODE_B' @@ -265,11 +265,11 @@ If the CLI is unavailable, run it inline: $(cat graphify-out/.graphify_python) -c " import json, sys import networkx as nx -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') term = 'NODE_NAME' term_lower = term.lower() diff --git a/graphify/skills/trae/references/transcribe.md b/graphify/skills/trae/references/transcribe.md index bbe8b93e45..b967f83799 100644 --- a/graphify/skills/trae/references/transcribe.md +++ b/graphify/skills/trae/references/transcribe.md @@ -8,11 +8,13 @@ Skip this step entirely if `detect` returned zero `video` files. Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. -**Strategy:** If `graphify-out/.graphify_analysis.json` exists from a previous run, read its top god-node labels and write a one-sentence domain hint from them. Otherwise use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` Pass the chosen prompt to Whisper as its initial prompt. No separate API call needed. +**Strategy:** Read the god nodes from `graphify-out/.graphify_detect.json` (or the analysis file if it exists from a previous run). You are already a language model — write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. + +**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` **Step 1 - Write the Whisper prompt yourself.** -Read the top god-node labels from previous analysis when available, then compose a short domain hint sentence. With no previous analysis, use the generic fallback above. +Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: - Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` - Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` diff --git a/graphify/skills/trae/references/update.md b/graphify/skills/trae/references/update.md index 058ca8e6d2..3632fd4126 100644 --- a/graphify/skills/trae/references/update.md +++ b/graphify/skills/trae/references/update.md @@ -12,7 +12,7 @@ import sys, json from graphify.detect import detect_incremental, save_manifest from pathlib import Path -result = detect_incremental(Path(sys.argv[1])) +result = detect_incremental(Path('INPUT_PATH')) new_total = result.get('new_total', 0) print(json.dumps(result, indent=2, ensure_ascii=False)) Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") @@ -24,7 +24,7 @@ if deleted: print(f'{len(deleted)} deleted file(s) to prune.') if new_total > 0: print(f'{new_total} new/changed file(s) to re-extract.') -" "INPUT_PATH" +" ``` Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: @@ -84,7 +84,7 @@ Then: ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from pathlib import Path from graphify.build import build_merge from graphify.detect import save_manifest @@ -113,7 +113,7 @@ G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, - root=sys.argv[1], + root='INPUT_PATH', directed=IS_DIRECTED, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -150,7 +150,7 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) # is lost forever (#2015). Mirrors the library extract path # (cli._stamped_manifest_files + clear_semantic + scan_corpus). from graphify.cli import _stamped_manifest_files -_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path(sys.argv[1])) +_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) # Changed semantic files dispatched this run but NOT stamped had their chunk fail # or be omitted; clear any stale semantic_hash so they are re-queued (#1948). _sem_types = ('document', 'paper', 'image') @@ -160,9 +160,9 @@ _cleared = _dispatched - _stamped # scan_corpus = the RAW full corpus so in-root files newly excluded since last run # are dropped rather than masquerading as deletions; untouched rows preserved (#1908). _scan = {f for fl in incremental['files'].values() for f in fl} -save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) print('[graphify update] Manifest saved.') -" "INPUT_PATH" +" ``` Then run Steps 4–8 on the merged graph as normal. @@ -174,7 +174,7 @@ $(cat graphify-out/.graphify_python) -c " import json from graphify.analyze import graph_diff from graphify.build import build_from_json -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph import networkx as nx from pathlib import Path @@ -184,7 +184,7 @@ new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(e G_new = build_from_json(new_extract, directed=IS_DIRECTED) if old_data: - G_old = load_node_link_graph(old_data) + G_old = json_graph.node_link_graph(old_data, edges='links') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: diff --git a/graphify/skills/vscode/references/add-watch.md b/graphify/skills/vscode/references/add-watch.md index fcbe0cd488..77844343e1 100644 --- a/graphify/skills/vscode/references/add-watch.md +++ b/graphify/skills/vscode/references/add-watch.md @@ -13,7 +13,7 @@ from graphify.ingest import ingest from pathlib import Path try: - out = ingest(sys.argv[1], Path('./raw'), author=sys.argv[2] or None, contributor=sys.argv[3] or None) + out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') print(f'Saved to {out}') except ValueError as e: print(f'error: {e}', file=sys.stderr) @@ -21,10 +21,10 @@ except ValueError as e: except RuntimeError as e: print(f'error: {e}', file=sys.stderr) sys.exit(1) -" "URL" "AUTHOR" "CONTRIBUTOR" +" ``` -Replace `URL` with the actual URL and pass the user's `AUTHOR` or `CONTRIBUTOR` when provided; use an empty string for either omitted value. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on the scan root recorded in `graphify-out/.graphify_root` to merge the new file into the existing graph. +Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. Supported URL types (auto-detected): - YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`) @@ -41,7 +41,7 @@ Supported URL types (auto-detected): Start a background watcher that monitors a folder and auto-updates the graph when files change. ```bash -$(cat graphify-out/.graphify_python) -m graphify.watch "INPUT_PATH" --debounce 3 +$(cat graphify-out/.graphify_python) -m graphify.watch INPUT_PATH --debounce 3 ``` Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: diff --git a/graphify/skills/vscode/references/exports.md b/graphify/skills/vscode/references/exports.md index 89ce996fb7..242ff868e0 100644 --- a/graphify/skills/vscode/references/exports.md +++ b/graphify/skills/vscode/references/exports.md @@ -20,27 +20,13 @@ graphify export wiki graphify export neo4j ``` -**If `--neo4j-push `** - push directly to a running Neo4j instance. Do not run these blocks through an agent tool: its shell is non-interactive and cannot read a hidden password. Show the block for the user's shell, ask them to run it in their own terminal, and wait for confirmation before continuing. Never ask them to paste the password into chat. - -POSIX shell: +**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: ```bash -NEO4J_PASSWORD="$("$(cat graphify-out/.graphify_python)" -c 'import getpass; print(getpass.getpass("Neo4j password: "))')" -export NEO4J_PASSWORD -graphify export neo4j --push bolt://localhost:7687 --user neo4j -unset NEO4J_PASSWORD -``` - -PowerShell: - -```powershell -$credential = Read-Host "Neo4j password" -AsSecureString -$env:NEO4J_PASSWORD = [System.Net.NetworkCredential]::new("", $credential).Password -graphify export neo4j --push bolt://localhost:7687 --user neo4j -Remove-Item Env:NEO4J_PASSWORD +graphify export neo4j --push bolt://localhost:7687 --user neo4j --password PASSWORD ``` -Default URI is `bolt://localhost:7687`, default user is `neo4j`. The command reads the password from `NEO4J_PASSWORD`. Uses MERGE - safe to re-run without creating duplicates. +Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. ### Step 7a - FalkorDB export (only if --falkordb or --falkordb-push flag) diff --git a/graphify/skills/vscode/references/extraction-spec.md b/graphify/skills/vscode/references/extraction-spec.md index a5a12a575e..388df7674f 100644 --- a/graphify/skills/vscode/references/extraction-spec.md +++ b/graphify/skills/vscode/references/extraction-spec.md @@ -61,7 +61,7 @@ confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a d Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is the **full repo-relative path with the extension dropped**, every path segment kept and joined with `_` (each segment lowercased with non-alphanumeric chars replaced by `_`), and entity is the symbol name similarly normalized. Use every directory level, not just the immediate parent — this keeps same-named files in different directories distinct. Examples: `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `lib_utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`; `docs/v1/api/README.md` + `getUser` → `docs_v1_api_readme_getuser`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or only the immediate parent (e.g., `auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project built under the old immediate-parent format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. Generate the extraction JSON matching this schema exactly: -{"nodes":[{"id":"src_auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} +{"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. diff --git a/graphify/skills/vscode/references/query.md b/graphify/skills/vscode/references/query.md index 6dd627863c..56565eb782 100644 --- a/graphify/skills/vscode/references/query.md +++ b/graphify/skills/vscode/references/query.md @@ -79,12 +79,12 @@ If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal ```bash $(cat graphify-out/.graphify_python) -c " import sys, json -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph import networkx as nx from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' @@ -197,11 +197,11 @@ If the CLI is unavailable, run it inline: $(cat graphify-out/.graphify_python) -c " import json, sys import networkx as nx -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') a_term = 'NODE_A' b_term = 'NODE_B' @@ -265,11 +265,11 @@ If the CLI is unavailable, run it inline: $(cat graphify-out/.graphify_python) -c " import json, sys import networkx as nx -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') term = 'NODE_NAME' term_lower = term.lower() diff --git a/graphify/skills/vscode/references/transcribe.md b/graphify/skills/vscode/references/transcribe.md index bbe8b93e45..b967f83799 100644 --- a/graphify/skills/vscode/references/transcribe.md +++ b/graphify/skills/vscode/references/transcribe.md @@ -8,11 +8,13 @@ Skip this step entirely if `detect` returned zero `video` files. Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. -**Strategy:** If `graphify-out/.graphify_analysis.json` exists from a previous run, read its top god-node labels and write a one-sentence domain hint from them. Otherwise use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` Pass the chosen prompt to Whisper as its initial prompt. No separate API call needed. +**Strategy:** Read the god nodes from `graphify-out/.graphify_detect.json` (or the analysis file if it exists from a previous run). You are already a language model — write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. + +**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` **Step 1 - Write the Whisper prompt yourself.** -Read the top god-node labels from previous analysis when available, then compose a short domain hint sentence. With no previous analysis, use the generic fallback above. +Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: - Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` - Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` diff --git a/graphify/skills/vscode/references/update.md b/graphify/skills/vscode/references/update.md index 058ca8e6d2..3632fd4126 100644 --- a/graphify/skills/vscode/references/update.md +++ b/graphify/skills/vscode/references/update.md @@ -12,7 +12,7 @@ import sys, json from graphify.detect import detect_incremental, save_manifest from pathlib import Path -result = detect_incremental(Path(sys.argv[1])) +result = detect_incremental(Path('INPUT_PATH')) new_total = result.get('new_total', 0) print(json.dumps(result, indent=2, ensure_ascii=False)) Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") @@ -24,7 +24,7 @@ if deleted: print(f'{len(deleted)} deleted file(s) to prune.') if new_total > 0: print(f'{new_total} new/changed file(s) to re-extract.') -" "INPUT_PATH" +" ``` Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: @@ -84,7 +84,7 @@ Then: ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from pathlib import Path from graphify.build import build_merge from graphify.detect import save_manifest @@ -113,7 +113,7 @@ G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, - root=sys.argv[1], + root='INPUT_PATH', directed=IS_DIRECTED, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -150,7 +150,7 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) # is lost forever (#2015). Mirrors the library extract path # (cli._stamped_manifest_files + clear_semantic + scan_corpus). from graphify.cli import _stamped_manifest_files -_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path(sys.argv[1])) +_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) # Changed semantic files dispatched this run but NOT stamped had their chunk fail # or be omitted; clear any stale semantic_hash so they are re-queued (#1948). _sem_types = ('document', 'paper', 'image') @@ -160,9 +160,9 @@ _cleared = _dispatched - _stamped # scan_corpus = the RAW full corpus so in-root files newly excluded since last run # are dropped rather than masquerading as deletions; untouched rows preserved (#1908). _scan = {f for fl in incremental['files'].values() for f in fl} -save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) print('[graphify update] Manifest saved.') -" "INPUT_PATH" +" ``` Then run Steps 4–8 on the merged graph as normal. @@ -174,7 +174,7 @@ $(cat graphify-out/.graphify_python) -c " import json from graphify.analyze import graph_diff from graphify.build import build_from_json -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph import networkx as nx from pathlib import Path @@ -184,7 +184,7 @@ new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(e G_new = build_from_json(new_extract, directed=IS_DIRECTED) if old_data: - G_old = load_node_link_graph(old_data) + G_old = json_graph.node_link_graph(old_data, edges='links') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: diff --git a/graphify/skills/windows/references/add-watch.md b/graphify/skills/windows/references/add-watch.md index fcbe0cd488..77844343e1 100644 --- a/graphify/skills/windows/references/add-watch.md +++ b/graphify/skills/windows/references/add-watch.md @@ -13,7 +13,7 @@ from graphify.ingest import ingest from pathlib import Path try: - out = ingest(sys.argv[1], Path('./raw'), author=sys.argv[2] or None, contributor=sys.argv[3] or None) + out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') print(f'Saved to {out}') except ValueError as e: print(f'error: {e}', file=sys.stderr) @@ -21,10 +21,10 @@ except ValueError as e: except RuntimeError as e: print(f'error: {e}', file=sys.stderr) sys.exit(1) -" "URL" "AUTHOR" "CONTRIBUTOR" +" ``` -Replace `URL` with the actual URL and pass the user's `AUTHOR` or `CONTRIBUTOR` when provided; use an empty string for either omitted value. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on the scan root recorded in `graphify-out/.graphify_root` to merge the new file into the existing graph. +Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. Supported URL types (auto-detected): - YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`) @@ -41,7 +41,7 @@ Supported URL types (auto-detected): Start a background watcher that monitors a folder and auto-updates the graph when files change. ```bash -$(cat graphify-out/.graphify_python) -m graphify.watch "INPUT_PATH" --debounce 3 +$(cat graphify-out/.graphify_python) -m graphify.watch INPUT_PATH --debounce 3 ``` Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: diff --git a/graphify/skills/windows/references/exports.md b/graphify/skills/windows/references/exports.md index 89ce996fb7..242ff868e0 100644 --- a/graphify/skills/windows/references/exports.md +++ b/graphify/skills/windows/references/exports.md @@ -20,27 +20,13 @@ graphify export wiki graphify export neo4j ``` -**If `--neo4j-push `** - push directly to a running Neo4j instance. Do not run these blocks through an agent tool: its shell is non-interactive and cannot read a hidden password. Show the block for the user's shell, ask them to run it in their own terminal, and wait for confirmation before continuing. Never ask them to paste the password into chat. - -POSIX shell: +**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: ```bash -NEO4J_PASSWORD="$("$(cat graphify-out/.graphify_python)" -c 'import getpass; print(getpass.getpass("Neo4j password: "))')" -export NEO4J_PASSWORD -graphify export neo4j --push bolt://localhost:7687 --user neo4j -unset NEO4J_PASSWORD -``` - -PowerShell: - -```powershell -$credential = Read-Host "Neo4j password" -AsSecureString -$env:NEO4J_PASSWORD = [System.Net.NetworkCredential]::new("", $credential).Password -graphify export neo4j --push bolt://localhost:7687 --user neo4j -Remove-Item Env:NEO4J_PASSWORD +graphify export neo4j --push bolt://localhost:7687 --user neo4j --password PASSWORD ``` -Default URI is `bolt://localhost:7687`, default user is `neo4j`. The command reads the password from `NEO4J_PASSWORD`. Uses MERGE - safe to re-run without creating duplicates. +Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. ### Step 7a - FalkorDB export (only if --falkordb or --falkordb-push flag) diff --git a/graphify/skills/windows/references/extraction-spec.md b/graphify/skills/windows/references/extraction-spec.md index a5a12a575e..388df7674f 100644 --- a/graphify/skills/windows/references/extraction-spec.md +++ b/graphify/skills/windows/references/extraction-spec.md @@ -61,7 +61,7 @@ confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a d Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is the **full repo-relative path with the extension dropped**, every path segment kept and joined with `_` (each segment lowercased with non-alphanumeric chars replaced by `_`), and entity is the symbol name similarly normalized. Use every directory level, not just the immediate parent — this keeps same-named files in different directories distinct. Examples: `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `lib_utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`; `docs/v1/api/README.md` + `getUser` → `docs_v1_api_readme_getuser`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or only the immediate parent (e.g., `auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project built under the old immediate-parent format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. Generate the extraction JSON matching this schema exactly: -{"nodes":[{"id":"src_auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} +{"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. diff --git a/graphify/skills/windows/references/query.md b/graphify/skills/windows/references/query.md index 6dd627863c..56565eb782 100644 --- a/graphify/skills/windows/references/query.md +++ b/graphify/skills/windows/references/query.md @@ -79,12 +79,12 @@ If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal ```bash $(cat graphify-out/.graphify_python) -c " import sys, json -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph import networkx as nx from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' @@ -197,11 +197,11 @@ If the CLI is unavailable, run it inline: $(cat graphify-out/.graphify_python) -c " import json, sys import networkx as nx -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') a_term = 'NODE_A' b_term = 'NODE_B' @@ -265,11 +265,11 @@ If the CLI is unavailable, run it inline: $(cat graphify-out/.graphify_python) -c " import json, sys import networkx as nx -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') term = 'NODE_NAME' term_lower = term.lower() diff --git a/graphify/skills/windows/references/transcribe.md b/graphify/skills/windows/references/transcribe.md index bbe8b93e45..b967f83799 100644 --- a/graphify/skills/windows/references/transcribe.md +++ b/graphify/skills/windows/references/transcribe.md @@ -8,11 +8,13 @@ Skip this step entirely if `detect` returned zero `video` files. Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. -**Strategy:** If `graphify-out/.graphify_analysis.json` exists from a previous run, read its top god-node labels and write a one-sentence domain hint from them. Otherwise use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` Pass the chosen prompt to Whisper as its initial prompt. No separate API call needed. +**Strategy:** Read the god nodes from `graphify-out/.graphify_detect.json` (or the analysis file if it exists from a previous run). You are already a language model — write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. + +**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` **Step 1 - Write the Whisper prompt yourself.** -Read the top god-node labels from previous analysis when available, then compose a short domain hint sentence. With no previous analysis, use the generic fallback above. +Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: - Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` - Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` diff --git a/graphify/skills/windows/references/update.md b/graphify/skills/windows/references/update.md index 058ca8e6d2..3632fd4126 100644 --- a/graphify/skills/windows/references/update.md +++ b/graphify/skills/windows/references/update.md @@ -12,7 +12,7 @@ import sys, json from graphify.detect import detect_incremental, save_manifest from pathlib import Path -result = detect_incremental(Path(sys.argv[1])) +result = detect_incremental(Path('INPUT_PATH')) new_total = result.get('new_total', 0) print(json.dumps(result, indent=2, ensure_ascii=False)) Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") @@ -24,7 +24,7 @@ if deleted: print(f'{len(deleted)} deleted file(s) to prune.') if new_total > 0: print(f'{new_total} new/changed file(s) to re-extract.') -" "INPUT_PATH" +" ``` Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: @@ -84,7 +84,7 @@ Then: ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from pathlib import Path from graphify.build import build_merge from graphify.detect import save_manifest @@ -113,7 +113,7 @@ G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, - root=sys.argv[1], + root='INPUT_PATH', directed=IS_DIRECTED, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -150,7 +150,7 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) # is lost forever (#2015). Mirrors the library extract path # (cli._stamped_manifest_files + clear_semantic + scan_corpus). from graphify.cli import _stamped_manifest_files -_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path(sys.argv[1])) +_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) # Changed semantic files dispatched this run but NOT stamped had their chunk fail # or be omitted; clear any stale semantic_hash so they are re-queued (#1948). _sem_types = ('document', 'paper', 'image') @@ -160,9 +160,9 @@ _cleared = _dispatched - _stamped # scan_corpus = the RAW full corpus so in-root files newly excluded since last run # are dropped rather than masquerading as deletions; untouched rows preserved (#1908). _scan = {f for fl in incremental['files'].values() for f in fl} -save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) print('[graphify update] Manifest saved.') -" "INPUT_PATH" +" ``` Then run Steps 4–8 on the merged graph as normal. @@ -174,7 +174,7 @@ $(cat graphify-out/.graphify_python) -c " import json from graphify.analyze import graph_diff from graphify.build import build_from_json -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph import networkx as nx from pathlib import Path @@ -184,7 +184,7 @@ new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(e G_new = build_from_json(new_extract, directed=IS_DIRECTED) if old_data: - G_old = load_node_link_graph(old_data) + G_old = json_graph.node_link_graph(old_data, edges='links') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: diff --git a/tests/test_skillgen.py b/tests/test_skillgen.py index bdde04c2e7..63a6f7c173 100644 --- a/tests/test_skillgen.py +++ b/tests/test_skillgen.py @@ -8,7 +8,6 @@ """ from __future__ import annotations -import json import sys from pathlib import Path @@ -59,157 +58,6 @@ def test_render_output_is_lf_only(): assert not art.content.endswith("\n\n"), art.path -def test_rendered_instructions_preserve_scan_root_and_runnable_commands(): - """Generated agent instructions must remain safe for real roots and flags.""" - platforms = gen.load_platforms() - artifacts = gen.render_all(platforms) - posix_cores = [ - artifact - for artifact in artifacts - if 'echo "$(cd' in artifact.content - ] - assert posix_cores - for artifact in posix_cores: - assert 'cd "INPUT_PATH"' in artifact.content, artifact.path - assert ( - "import sys, json\n" - "from graphify.extract import collect_files, extract\n" - "from pathlib import Path\n" - "import json\n" - ) not in artifact.content, artifact.path - assert "graphify export html --no-viz" not in artifact.content, artifact.path - - windows_core = next( - artifact - for artifact in artifacts - if artifact.path == "graphify/skill-windows.md" - ) - assert '(Resolve-Path "INPUT_PATH").Path' in windows_core.content - - for artifact in artifacts: - assert '"id":"auth_session_validatetoken"' not in artifact.content, artifact.path - assert "--password PASSWORD" not in artifact.content, artifact.path - assert "graphify-out/.needs_update" not in artifact.content, artifact.path - for line in artifact.content.splitlines(): - if line.startswith('{"nodes":'): - json.loads(line) - - direct_loaders = [ - artifact.path - for artifact in artifacts - if "json_graph.node_link_graph(" in artifact.content - ] - assert direct_loaders == [] - assert any("load_node_link_graph(" in artifact.content for artifact in artifacts) - - watch_artifacts = [ - artifact - for artifact in artifacts - if "graphify.watch" in artifact.content and "INPUT_PATH" in artifact.content - ] - assert watch_artifacts - for artifact in watch_artifacts: - assert 'graphify.watch "INPUT_PATH"' in artifact.content, artifact.path - assert "graphify.watch INPUT_PATH" not in artifact.content, artifact.path - - for artifact in artifacts: - assert "'INPUT_PATH'" not in artifact.content, artifact.path - assert "'SPEC_PATH'" not in artifact.content, artifact.path - - ingest_artifacts = [ - artifact - for artifact in artifacts - if "from graphify.ingest import ingest" in artifact.content - ] - assert ingest_artifacts - for artifact in ingest_artifacts: - assert ( - "out = ingest(sys.argv[1], Path('./raw'), author=sys.argv[2] or None, " - "contributor=sys.argv[3] or None)" - ) in artifact.content, artifact.path - assert "ingest('URL'" not in artifact.content, artifact.path - argv_close = ( - gen._PY_INVOKE_PS_CLOSE + ' "URL" "AUTHOR" "CONTRIBUTOR"' - if artifact.path == "graphify/skill-windows.md" - else '" "URL" "AUTHOR" "CONTRIBUTOR"' - ) - assert argv_close in artifact.content, artifact.path - - - agents_core = next( - artifact - for artifact in artifacts - if artifact.path == "graphify/skill-agents.md" - ) - assert "You MUST use the Agent tool here" not in agents_core.content - assert "After each Agent call completes" not in agents_core.content - assert "You MUST use the subagent tool here" in agents_core.content - assert "After each subagent call completes" in agents_core.content - for command in ( - "export obsidian", - "export html", - 'query ""', - ): - assert ( - f"$(cat graphify-out/.graphify_python) -m graphify {command}" - in agents_core.content - ), command - assert ( - f"& (Get-Content graphify-out\\.graphify_python) -m graphify {command}" - in windows_core.content - ), command - - transcription_artifacts = [ - artifact - for artifact in artifacts - if artifact.path.endswith("/references/transcribe.md") - or artifact.path in {"graphify/skill-aider.md", "graphify/skill-devin.md"} - ] - assert transcription_artifacts - for artifact in transcription_artifacts: - assert "god nodes from" not in artifact.content, artifact.path - assert "previous analysis" in artifact.content, artifact.path - - - exports = [ - artifact - for artifact in artifacts - if artifact.path.endswith("/references/exports.md") - ] - assert exports - for artifact in exports: - assert "getpass.getpass" in artifact.content, artifact.path - assert 'Read-Host "Neo4j password" -AsSecureString' in artifact.content, artifact.path - assert "Do not run these blocks through an agent tool" in artifact.content, artifact.path - - labeling_cores = [ - artifact - for artifact in artifacts - if "# Re-export so graph.json nodes carry the curated community_name" in artifact.content - ] - assert labeling_cores - for artifact in labeling_cores: - step_five = artifact.content.split("### Step 5", 1)[1].split("### Step 6", 1)[0] - write_graph = step_five.index("wrote = to_json(") - shrink_guard = step_five.index("if not wrote:") - abort = step_five.index("raise SystemExit(1)", shrink_guard) - write_report = step_five.index("GRAPH_REPORT.md") - write_labels = step_five.index("graphify_labels.json") - assert write_graph < shrink_guard < abort < write_report < write_labels, artifact.path - - add_watch = [ - artifact - for artifact in artifacts - if artifact.path.endswith("/references/add-watch.md") - ] - assert add_watch - for artifact in add_watch: - assert ( - "run the `--update` pipeline on the scan root recorded in " - "`graphify-out/.graphify_root`" - ) in artifact.content, artifact.path - - def test_no_version_or_timestamp_in_output(): """No generated artifact carries the package version string.""" from graphify.__main__ import __version__ @@ -584,7 +432,7 @@ def test_windows_python_step_bodies_match_posix_verbatim(): for line in claude_core.splitlines(): if line == gen._PY_INVOKE_POSIX: current = [] - elif current is not None and line in gen._PY_CLOSE_TRANSLATIONS: + elif current is not None and line == '"': bodies.append("\n".join(gen._unescape_bash_dq(l) for l in current)) current = None elif current is not None: @@ -612,21 +460,6 @@ def test_powershell_translator_rejects_unknown_bash(): assert gen._translate_bash_block([gen._FIND_CHUNKS_POSIX]) == [gen._FIND_CHUNKS_PS] -def test_powershell_translator_preserves_python_argv(): - """Path placeholders remain shell arguments when Python becomes a here-string.""" - assert gen._translate_bash_block([ - gen._PY_INVOKE_POSIX, - "import sys", - "print(sys.argv[1])", - '" "INPUT_PATH"', - ]) == [ - gen._PY_INVOKE_PS_OPEN, - "import sys", - "print(sys.argv[1])", - gen._PY_INVOKE_PS_CLOSE + ' "INPUT_PATH"', - ] - - def test_posix_hosts_keep_their_bash_invocations(): """The translation is scoped to powershell-shell hosts: the POSIX core keeps its ``$(cat ...) -c`` blocks and bash fences byte-for-byte.""" @@ -1254,18 +1087,12 @@ def test_semantic_cache_calls_pass_prompt_file_for_every_split_host(): for a in bodies: for call in ("check_semantic_cache(", "save_semantic_cache("): line = next(ln for ln in a.content.splitlines() if call in ln and "import" not in ln) - assert "prompt_file=sys.argv[2]" in line, ( - f"{a.path}: {call} must read the extraction prompt from argv " - f"(#1939) — got: {line.strip()}" + assert "prompt_file='SPEC_PATH'" in line, ( + f"{a.path}: {call} must pass prompt_file so entries are attributed " + f"to the extraction prompt (#1939) — got: {line.strip()}" ) # The placeholder is inert unless the body tells the agent what to substitute. assert "SPEC_PATH below is the **absolute** path" in a.content, a.path - argv_close = ( - gen._PY_INVOKE_PS_CLOSE + ' "INPUT_PATH" "SPEC_PATH"' - if a.path == "graphify/skill-windows.md" - else '" "INPUT_PATH" "SPEC_PATH"' - ) - assert a.content.count(argv_close) == 2, a.path def test_windows_skill_writes_marker_files_without_a_bom(): diff --git a/tools/skillgen/expected/graphify__skill-agents.md b/tools/skillgen/expected/graphify__skill-agents.md index 02be4d1f6f..190827d9ac 100644 --- a/tools/skillgen/expected/graphify__skill-agents.md +++ b/tools/skillgen/expected/graphify__skill-agents.md @@ -97,7 +97,7 @@ fi mkdir -p graphify-out "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" # Save scan root so `graphify update` (no args) knows where to look next time -echo "$(cd "INPUT_PATH" && pwd)" > graphify-out/.graphify_root +echo "$(cd INPUT_PATH && pwd)" > graphify-out/.graphify_root ``` If the import succeeds, print nothing and move straight to Step 2. @@ -108,15 +108,15 @@ If the import succeeds, print nothing and move straight to Step 2. ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from graphify.detect import detect from pathlib import Path -result = detect(Path(sys.argv[1])) +result = detect(Path('INPUT_PATH')) # Write the sidecar from Python, not a shell redirect, so the same block renders # on PowerShell hosts without console-encoding drift (#2528). Path('graphify-out/.graphify_detect.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") print(f'Detected {result[\"total_files\"]} files') -" "INPUT_PATH" +" ``` Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: @@ -176,6 +176,7 @@ $(cat graphify-out/.graphify_python) -c " import sys, json from graphify.extract import collect_files, extract from pathlib import Path +import json code_files = [] detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) @@ -183,13 +184,13 @@ for f in detect.get('files', {}).get('code', []): code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) if code_files: - result = extract(code_files, cache_root=Path(sys.argv[1])) + result = extract(code_files, cache_root=Path('INPUT_PATH')) Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') else: Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") print('No code files - skipping AST extraction') -" "INPUT_PATH" +" ``` #### Part B - Semantic extraction (parallel subagents) @@ -204,7 +205,7 @@ Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],' " ``` -**MANDATORY: You MUST use the subagent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the subagent tool you are doing this wrong.** +**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** Before dispatching subagents, print a timing estimate: - Load `total_words` and file counts from `graphify-out/.graphify_detect.json` @@ -220,7 +221,7 @@ SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from graphify.cache import check_semantic_cache from pathlib import Path @@ -230,7 +231,7 @@ detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encodin # every source file (#1392). Video is transcribed to a document in Step 2.5 first. all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root=sys.argv[1], prompt_file=sys.argv[2]) +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') # Always (re)write the cache file: write hits, else DELETE any leftover from a prior # run so Part C never merges a stale .graphify_cached.json (#1392). @@ -240,7 +241,7 @@ else: Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" "INPUT_PATH" "SPEC_PATH" +" ``` Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. @@ -282,7 +283,7 @@ Wait for all subagents. For each result: If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. -Merge all chunk files into `.graphify_semantic_new.json`. **After each subagent call completes, read the real token counts from the subagent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: +Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: ```bash $(cat graphify-out/.graphify_python) -c " import json, glob @@ -309,15 +310,15 @@ print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens' Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root=sys.argv[1], allowed_source_files=uncached, prompt_file=sys.argv[2]) +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') print(f'Cached {saved} files') -" "INPUT_PATH" "SPEC_PATH" +" ``` Merge cached + new results into `graphify-out/.graphify_semantic.json`: @@ -406,7 +407,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -431,7 +432,7 @@ if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') print('If this shrink is intentional (you deleted files), re-run a full build with --force.') raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, sys.argv[1], suggested_questions=questions) +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") analysis = { 'communities': {str(k): v for k, v in communities.items()}, @@ -442,7 +443,7 @@ analysis = { } Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" "INPUT_PATH" +" ``` If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. @@ -455,12 +456,12 @@ A non-destructive diagnostic on the extraction, before labeling. It surfaces edg ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from pathlib import Path from graphify.diagnostics import diagnose_extraction, format_diagnostic_report extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root=sys.argv[1]) +summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( ('dangling_endpoint_edges', 'dangling-endpoint edges'), @@ -470,7 +471,7 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0)] print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') -" "INPUT_PATH" +" ``` Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules). @@ -496,7 +497,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} @@ -507,7 +508,9 @@ labels = LABELS_DICT # Regenerate questions with real community labels (labels affect question phrasing) questions = suggest_questions(G, communities, labels) -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, sys.argv[1], suggested_questions=questions) +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") +Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") # Re-export so graph.json nodes carry the curated community_name (#2490). # Same extraction as Step 4, so the #479 shrink-guard passes on node count; # if it still refuses, surface the guard message - do not force past it. @@ -515,11 +518,8 @@ wrote = to_json(G, communities, 'graphify-out/graph.json', community_labels=labe if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') print('If this shrink is intentional (you deleted files), re-run a full build with --force.') - raise SystemExit(1) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") print('Report updated with community labels') -" "INPUT_PATH" +" ``` Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). @@ -534,14 +534,15 @@ If `--obsidian` was given: - If `--obsidian-dir ` was also given, pass it via `--dir`. Otherwise defaults to `graphify-out/obsidian`. ```bash -$(cat graphify-out/.graphify_python) -m graphify export obsidian -# Add `--dir ~/vaults/my-project` to use a custom directory. +graphify export obsidian +# or with custom dir: graphify export obsidian --dir ~/vaults/my-project ``` Generate the HTML graph (always, unless `--no-viz`): ```bash -$(cat graphify-out/.graphify_python) -m graphify export html # auto-aggregates to community view if graph > 5000 nodes +graphify export html # auto-aggregates to community view if graph > 5000 nodes +# or: graphify export html --no-viz ``` ### Steps 6b-8 - Wiki, Neo4j, FalkorDB, SVG, GraphML, MCP, benchmark (only on their flags) @@ -554,7 +555,7 @@ These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, ` ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from pathlib import Path from datetime import datetime, timezone from graphify.detect import save_manifest @@ -577,7 +578,7 @@ extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encod # types are gated on output. from graphify.cli import _stamped_manifest_files _corpus = detect.get('all_files') or detect['files'] -_manifest_files = _stamped_manifest_files(_corpus, extract, Path(sys.argv[1])) +_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) # Files dispatched this run (the changed subset) but NOT stamped above still carry # a stale semantic_hash from a prior run; clear it so detect_incremental re-queues # them instead of reading them as unchanged (#1948). @@ -589,7 +590,7 @@ _cleared = _dispatched - _stamped # files newly excluded since last run are dropped rather than masquerading as # deletions; untouched files' prior rows are still preserved (#1908). _scan = {f for fl in _corpus.values() for f in fl} -save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) # Update cumulative cost tracker input_tok = extract.get('input_tokens', 0) @@ -613,10 +614,10 @@ cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\" print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" "INPUT_PATH" +" rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null -rm -f graphify-out/needs_update 2>/dev/null || true +rm -f graphify-out/.needs_update 2>/dev/null || true ``` Replace INPUT_PATH with the actual path (same value used in Steps 4-5) so the manifest is relativized to the scan root. @@ -681,7 +682,7 @@ Both are non-default subcommands. `--update` re-extracts only new or changed fil When `graphify-out/graph.json` already exists and the user asks a question about the corpus, answer from the graph rather than rebuilding it: ```bash -$(cat graphify-out/.graphify_python) -m graphify query "" +graphify query "" ``` Before traversal, expand the question against the graph's own vocabulary so a wording mismatch does not collapse the answer to noise. If the `graphify query` CLI is unavailable, fall back to an inline NetworkX traversal of `graphify-out/graph.json`. Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. For that vocab-expansion step, the BFS/DFS traversal modes, the `--budget` cap, the NetworkX fallback, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`. diff --git a/tools/skillgen/expected/graphify__skill-aider.md b/tools/skillgen/expected/graphify__skill-aider.md index aacd9f31fc..4996beb787 100644 --- a/tools/skillgen/expected/graphify__skill-aider.md +++ b/tools/skillgen/expected/graphify__skill-aider.md @@ -101,12 +101,11 @@ If the import succeeds, print nothing and move straight to Step 2. ```bash $(cat graphify-out/.graphify_python) -c " import json -import sys from graphify.detect import detect from pathlib import Path -result = detect(Path(sys.argv[1])) +result = detect(Path('INPUT_PATH')) print(json.dumps(result)) -" "INPUT_PATH" > .graphify_detect.json +" > .graphify_detect.json ``` Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: @@ -134,11 +133,13 @@ Skip this step entirely if `detect` returned zero `video` files. Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. -**Strategy:** If an analysis file exists from a previous run, read its top god-node labels and write a one-sentence domain hint from them. Otherwise use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` Pass the chosen prompt to Whisper as its initial prompt. No separate API call needed. +**Strategy:** Read the god nodes from the detect output or analysis file. You are already a language model - write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. + +**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` **Step 1 - Write the Whisper prompt yourself.** -Read the top god-node labels from previous analysis when available, then compose a short domain hint sentence. With no previous analysis, use the generic fallback above. +Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: - Labels: `transformer, attention, encoder, decoder` -> `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` - Labels: `kubernetes, deployment, pod, helm` -> `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` @@ -425,7 +426,7 @@ wrote = to_json(G, communities, 'graphify-out/graph.json') if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (fewer nodes than the existing graph). Run a full rebuild to be safe.') raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, sys.argv[1], suggested_questions=questions) +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) Path('graphify-out/GRAPH_REPORT.md').write_text(report) analysis = { @@ -437,7 +438,7 @@ analysis = { } Path('.graphify_analysis.json').write_text(json.dumps(analysis, indent=2)) print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" "INPUT_PATH" +" ``` If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. @@ -475,18 +476,17 @@ labels = LABELS_DICT # Regenerate questions with real community labels (labels affect question phrasing) questions = suggest_questions(G, communities, labels) -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, sys.argv[1], suggested_questions=questions) +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report) +Path('.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()})) # Re-export so graph.json nodes carry the curated community_name (#2490). # Same extraction as Step 4, so the #479 shrink-guard passes on node count; # if it still refuses, surface the guard message - do not force past it. wrote = to_json(G, communities, 'graphify-out/graph.json', community_labels=labels) if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (fewer nodes than the existing graph). Run a full rebuild to be safe.') - raise SystemExit(1) -Path('graphify-out/GRAPH_REPORT.md').write_text(report) -Path('.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()})) print('Report updated with community labels') -" "INPUT_PATH" +" ``` Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). @@ -678,7 +678,6 @@ Print the output directly in chat. If `total_words <= 5000`, skip silently - the ```bash $(cat graphify-out/.graphify_python) -c " import json -import sys from pathlib import Path from datetime import datetime, timezone from graphify.detect import save_manifest @@ -689,13 +688,13 @@ extract = json.loads(Path('.graphify_extract.json').read_text()) # Stamp only semantic files that produced output so a failed chunk is re-queued next run, not lost (#2015). from graphify.cli import _stamped_manifest_files _corpus = detect.get('all_files') or detect['files'] -_manifest_files = _stamped_manifest_files(_corpus, extract, Path(sys.argv[1])) +_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) _sem_types = ('document', 'paper', 'image') _dispatched = {f for t, fl in detect['files'].items() if t in _sem_types for f in fl} _stamped = {f for fl in _manifest_files.values() for f in fl} _cleared = _dispatched - _stamped _scan = {f for fl in _corpus.values() for f in fl} -save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) # Update cumulative cost tracker input_tok = extract.get('input_tokens', 0) @@ -719,9 +718,9 @@ cost_path.write_text(json.dumps(cost, indent=2)) print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" "INPUT_PATH" +" rm -f .graphify_detect.json .graphify_extract.json .graphify_ast.json .graphify_semantic.json .graphify_analysis.json .graphify_labels.json; find . -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null -rm -f graphify-out/needs_update 2>/dev/null || true +rm -f graphify-out/.needs_update 2>/dev/null || true ``` Tell the user (omit the obsidian line unless --obsidian was given): @@ -765,7 +764,7 @@ import sys, json from graphify.detect import detect_incremental, save_manifest from pathlib import Path -result = detect_incremental(Path(sys.argv[1])) +result = detect_incremental(Path('INPUT_PATH')) new_total = result.get('new_total', 0) print(json.dumps(result, indent=2)) Path('.graphify_incremental.json').write_text(json.dumps(result)) @@ -777,7 +776,7 @@ if deleted: print(f'{len(deleted)} deleted file(s) to prune.') if new_total > 0: print(f'{new_total} new/changed file(s) to re-extract.') -" "INPUT_PATH" +" ``` If new files exist, first check whether all changed files are code files: @@ -822,13 +821,13 @@ $(cat graphify-out/.graphify_python) -c " import sys, json from graphify.build import build_from_json from graphify.export import to_json -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph import networkx as nx from pathlib import Path # Load existing graph existing_data = json.loads(Path('graphify-out/graph.json').read_text()) -G_existing = load_node_link_graph(existing_data) +G_existing = json_graph.node_link_graph(existing_data, edges='links') # Load new extraction new_extraction = json.loads(Path('.graphify_extract.json').read_text()) @@ -849,7 +848,7 @@ $(cat graphify-out/.graphify_python) -c " import json from graphify.analyze import graph_diff from graphify.build import build_from_json -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph import networkx as nx from pathlib import Path @@ -859,7 +858,7 @@ new_extract = json.loads(Path('.graphify_extract.json').read_text()) G_new = build_from_json(new_extract, directed=IS_DIRECTED) if old_data: - G_old = load_node_link_graph(old_data) + G_old = json_graph.node_link_graph(old_data, edges='links') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: @@ -885,12 +884,12 @@ from graphify.cluster import cluster, score_all from graphify.analyze import god_nodes, surprising_connections from graphify.report import generate from graphify.export import to_json -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph import networkx as nx from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text()) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') detection = {'total_files': 0, 'total_words': 99999, 'needs_graph': True, 'warning': None, 'files': {'code': [], 'document': [], 'paper': []}} @@ -954,12 +953,12 @@ Load `graphify-out/graph.json`, then: ```bash $(cat graphify-out/.graphify_python) -c " import sys, json -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph import networkx as nx from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text()) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' @@ -1069,11 +1068,11 @@ If it fails, stop and tell the user to run `/graphify ` first. $(cat graphify-out/.graphify_python) -c " import json, sys import networkx as nx -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text()) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') a_term = 'NODE_A' b_term = 'NODE_B' @@ -1142,11 +1141,11 @@ If it fails, stop and tell the user to run `/graphify ` first. $(cat graphify-out/.graphify_python) -c " import json, sys import networkx as nx -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text()) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') term = 'NODE_NAME' term_lower = term.lower() @@ -1200,7 +1199,7 @@ from graphify.ingest import ingest from pathlib import Path try: - out = ingest(sys.argv[1], Path('./raw'), author=sys.argv[2] or None, contributor=sys.argv[3] or None) + out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') print(f'Saved to {out}') except ValueError as e: print(f'error: {e}', file=sys.stderr) @@ -1208,10 +1207,10 @@ except ValueError as e: except RuntimeError as e: print(f'error: {e}', file=sys.stderr) sys.exit(1) -" "URL" "AUTHOR" "CONTRIBUTOR" +" ``` -Replace `URL` with the actual URL and pass the user's `AUTHOR` or `CONTRIBUTOR` when provided; use an empty string for either omitted value. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. +Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. Supported URL types (auto-detected): - Twitter/X → fetched via oEmbed, saved as `.md` with tweet text and author @@ -1227,7 +1226,7 @@ Supported URL types (auto-detected): Start a background watcher that monitors a folder and auto-updates the graph when files change. ```bash -python3 -m graphify.watch "INPUT_PATH" --debounce 3 +python3 -m graphify.watch INPUT_PATH --debounce 3 ``` Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: diff --git a/tools/skillgen/expected/graphify__skill-amp.md b/tools/skillgen/expected/graphify__skill-amp.md index 02be4d1f6f..190827d9ac 100644 --- a/tools/skillgen/expected/graphify__skill-amp.md +++ b/tools/skillgen/expected/graphify__skill-amp.md @@ -97,7 +97,7 @@ fi mkdir -p graphify-out "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" # Save scan root so `graphify update` (no args) knows where to look next time -echo "$(cd "INPUT_PATH" && pwd)" > graphify-out/.graphify_root +echo "$(cd INPUT_PATH && pwd)" > graphify-out/.graphify_root ``` If the import succeeds, print nothing and move straight to Step 2. @@ -108,15 +108,15 @@ If the import succeeds, print nothing and move straight to Step 2. ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from graphify.detect import detect from pathlib import Path -result = detect(Path(sys.argv[1])) +result = detect(Path('INPUT_PATH')) # Write the sidecar from Python, not a shell redirect, so the same block renders # on PowerShell hosts without console-encoding drift (#2528). Path('graphify-out/.graphify_detect.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") print(f'Detected {result[\"total_files\"]} files') -" "INPUT_PATH" +" ``` Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: @@ -176,6 +176,7 @@ $(cat graphify-out/.graphify_python) -c " import sys, json from graphify.extract import collect_files, extract from pathlib import Path +import json code_files = [] detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) @@ -183,13 +184,13 @@ for f in detect.get('files', {}).get('code', []): code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) if code_files: - result = extract(code_files, cache_root=Path(sys.argv[1])) + result = extract(code_files, cache_root=Path('INPUT_PATH')) Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') else: Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") print('No code files - skipping AST extraction') -" "INPUT_PATH" +" ``` #### Part B - Semantic extraction (parallel subagents) @@ -204,7 +205,7 @@ Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],' " ``` -**MANDATORY: You MUST use the subagent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the subagent tool you are doing this wrong.** +**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** Before dispatching subagents, print a timing estimate: - Load `total_words` and file counts from `graphify-out/.graphify_detect.json` @@ -220,7 +221,7 @@ SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from graphify.cache import check_semantic_cache from pathlib import Path @@ -230,7 +231,7 @@ detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encodin # every source file (#1392). Video is transcribed to a document in Step 2.5 first. all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root=sys.argv[1], prompt_file=sys.argv[2]) +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') # Always (re)write the cache file: write hits, else DELETE any leftover from a prior # run so Part C never merges a stale .graphify_cached.json (#1392). @@ -240,7 +241,7 @@ else: Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" "INPUT_PATH" "SPEC_PATH" +" ``` Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. @@ -282,7 +283,7 @@ Wait for all subagents. For each result: If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. -Merge all chunk files into `.graphify_semantic_new.json`. **After each subagent call completes, read the real token counts from the subagent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: +Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: ```bash $(cat graphify-out/.graphify_python) -c " import json, glob @@ -309,15 +310,15 @@ print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens' Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root=sys.argv[1], allowed_source_files=uncached, prompt_file=sys.argv[2]) +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') print(f'Cached {saved} files') -" "INPUT_PATH" "SPEC_PATH" +" ``` Merge cached + new results into `graphify-out/.graphify_semantic.json`: @@ -406,7 +407,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -431,7 +432,7 @@ if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') print('If this shrink is intentional (you deleted files), re-run a full build with --force.') raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, sys.argv[1], suggested_questions=questions) +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") analysis = { 'communities': {str(k): v for k, v in communities.items()}, @@ -442,7 +443,7 @@ analysis = { } Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" "INPUT_PATH" +" ``` If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. @@ -455,12 +456,12 @@ A non-destructive diagnostic on the extraction, before labeling. It surfaces edg ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from pathlib import Path from graphify.diagnostics import diagnose_extraction, format_diagnostic_report extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root=sys.argv[1]) +summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( ('dangling_endpoint_edges', 'dangling-endpoint edges'), @@ -470,7 +471,7 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0)] print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') -" "INPUT_PATH" +" ``` Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules). @@ -496,7 +497,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} @@ -507,7 +508,9 @@ labels = LABELS_DICT # Regenerate questions with real community labels (labels affect question phrasing) questions = suggest_questions(G, communities, labels) -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, sys.argv[1], suggested_questions=questions) +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") +Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") # Re-export so graph.json nodes carry the curated community_name (#2490). # Same extraction as Step 4, so the #479 shrink-guard passes on node count; # if it still refuses, surface the guard message - do not force past it. @@ -515,11 +518,8 @@ wrote = to_json(G, communities, 'graphify-out/graph.json', community_labels=labe if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') print('If this shrink is intentional (you deleted files), re-run a full build with --force.') - raise SystemExit(1) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") print('Report updated with community labels') -" "INPUT_PATH" +" ``` Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). @@ -534,14 +534,15 @@ If `--obsidian` was given: - If `--obsidian-dir ` was also given, pass it via `--dir`. Otherwise defaults to `graphify-out/obsidian`. ```bash -$(cat graphify-out/.graphify_python) -m graphify export obsidian -# Add `--dir ~/vaults/my-project` to use a custom directory. +graphify export obsidian +# or with custom dir: graphify export obsidian --dir ~/vaults/my-project ``` Generate the HTML graph (always, unless `--no-viz`): ```bash -$(cat graphify-out/.graphify_python) -m graphify export html # auto-aggregates to community view if graph > 5000 nodes +graphify export html # auto-aggregates to community view if graph > 5000 nodes +# or: graphify export html --no-viz ``` ### Steps 6b-8 - Wiki, Neo4j, FalkorDB, SVG, GraphML, MCP, benchmark (only on their flags) @@ -554,7 +555,7 @@ These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, ` ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from pathlib import Path from datetime import datetime, timezone from graphify.detect import save_manifest @@ -577,7 +578,7 @@ extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encod # types are gated on output. from graphify.cli import _stamped_manifest_files _corpus = detect.get('all_files') or detect['files'] -_manifest_files = _stamped_manifest_files(_corpus, extract, Path(sys.argv[1])) +_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) # Files dispatched this run (the changed subset) but NOT stamped above still carry # a stale semantic_hash from a prior run; clear it so detect_incremental re-queues # them instead of reading them as unchanged (#1948). @@ -589,7 +590,7 @@ _cleared = _dispatched - _stamped # files newly excluded since last run are dropped rather than masquerading as # deletions; untouched files' prior rows are still preserved (#1908). _scan = {f for fl in _corpus.values() for f in fl} -save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) # Update cumulative cost tracker input_tok = extract.get('input_tokens', 0) @@ -613,10 +614,10 @@ cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\" print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" "INPUT_PATH" +" rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null -rm -f graphify-out/needs_update 2>/dev/null || true +rm -f graphify-out/.needs_update 2>/dev/null || true ``` Replace INPUT_PATH with the actual path (same value used in Steps 4-5) so the manifest is relativized to the scan root. @@ -681,7 +682,7 @@ Both are non-default subcommands. `--update` re-extracts only new or changed fil When `graphify-out/graph.json` already exists and the user asks a question about the corpus, answer from the graph rather than rebuilding it: ```bash -$(cat graphify-out/.graphify_python) -m graphify query "" +graphify query "" ``` Before traversal, expand the question against the graph's own vocabulary so a wording mismatch does not collapse the answer to noise. If the `graphify query` CLI is unavailable, fall back to an inline NetworkX traversal of `graphify-out/graph.json`. Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. For that vocab-expansion step, the BFS/DFS traversal modes, the `--budget` cap, the NetworkX fallback, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`. diff --git a/tools/skillgen/expected/graphify__skill-claw.md b/tools/skillgen/expected/graphify__skill-claw.md index 55b77d6828..abd2811d23 100644 --- a/tools/skillgen/expected/graphify__skill-claw.md +++ b/tools/skillgen/expected/graphify__skill-claw.md @@ -97,7 +97,7 @@ fi mkdir -p graphify-out "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" # Save scan root so `graphify update` (no args) knows where to look next time -echo "$(cd "INPUT_PATH" && pwd)" > graphify-out/.graphify_root +echo "$(cd INPUT_PATH && pwd)" > graphify-out/.graphify_root ``` If the import succeeds, print nothing and move straight to Step 2. @@ -108,15 +108,15 @@ If the import succeeds, print nothing and move straight to Step 2. ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from graphify.detect import detect from pathlib import Path -result = detect(Path(sys.argv[1])) +result = detect(Path('INPUT_PATH')) # Write the sidecar from Python, not a shell redirect, so the same block renders # on PowerShell hosts without console-encoding drift (#2528). Path('graphify-out/.graphify_detect.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") print(f'Detected {result[\"total_files\"]} files') -" "INPUT_PATH" +" ``` Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: @@ -176,6 +176,7 @@ $(cat graphify-out/.graphify_python) -c " import sys, json from graphify.extract import collect_files, extract from pathlib import Path +import json code_files = [] detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) @@ -183,13 +184,13 @@ for f in detect.get('files', {}).get('code', []): code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) if code_files: - result = extract(code_files, cache_root=Path(sys.argv[1])) + result = extract(code_files, cache_root=Path('INPUT_PATH')) Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') else: Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") print('No code files - skipping AST extraction') -" "INPUT_PATH" +" ``` #### Part B - Semantic extraction (parallel subagents) @@ -204,7 +205,7 @@ Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],' " ``` -**MANDATORY: You MUST use the subagent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the subagent tool you are doing this wrong.** +**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** Before dispatching subagents, print a timing estimate: - Load `total_words` and file counts from `graphify-out/.graphify_detect.json` @@ -220,7 +221,7 @@ SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from graphify.cache import check_semantic_cache from pathlib import Path @@ -230,7 +231,7 @@ detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encodin # every source file (#1392). Video is transcribed to a document in Step 2.5 first. all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root=sys.argv[1], prompt_file=sys.argv[2]) +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') # Always (re)write the cache file: write hits, else DELETE any leftover from a prior # run so Part C never merges a stale .graphify_cached.json (#1392). @@ -240,7 +241,7 @@ else: Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" "INPUT_PATH" "SPEC_PATH" +" ``` Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. @@ -285,7 +286,7 @@ Wait for all subagents. For each result: If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. -Merge all chunk files into `.graphify_semantic_new.json`. **After each subagent call completes, read the real token counts from the subagent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: +Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: ```bash $(cat graphify-out/.graphify_python) -c " import json, glob @@ -312,15 +313,15 @@ print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens' Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root=sys.argv[1], allowed_source_files=uncached, prompt_file=sys.argv[2]) +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') print(f'Cached {saved} files') -" "INPUT_PATH" "SPEC_PATH" +" ``` Merge cached + new results into `graphify-out/.graphify_semantic.json`: @@ -409,7 +410,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -434,7 +435,7 @@ if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') print('If this shrink is intentional (you deleted files), re-run a full build with --force.') raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, sys.argv[1], suggested_questions=questions) +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") analysis = { 'communities': {str(k): v for k, v in communities.items()}, @@ -445,7 +446,7 @@ analysis = { } Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" "INPUT_PATH" +" ``` If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. @@ -458,12 +459,12 @@ A non-destructive diagnostic on the extraction, before labeling. It surfaces edg ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from pathlib import Path from graphify.diagnostics import diagnose_extraction, format_diagnostic_report extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root=sys.argv[1]) +summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( ('dangling_endpoint_edges', 'dangling-endpoint edges'), @@ -473,7 +474,7 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0)] print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') -" "INPUT_PATH" +" ``` Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules). @@ -499,7 +500,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} @@ -510,7 +511,9 @@ labels = LABELS_DICT # Regenerate questions with real community labels (labels affect question phrasing) questions = suggest_questions(G, communities, labels) -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, sys.argv[1], suggested_questions=questions) +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") +Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") # Re-export so graph.json nodes carry the curated community_name (#2490). # Same extraction as Step 4, so the #479 shrink-guard passes on node count; # if it still refuses, surface the guard message - do not force past it. @@ -518,11 +521,8 @@ wrote = to_json(G, communities, 'graphify-out/graph.json', community_labels=labe if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') print('If this shrink is intentional (you deleted files), re-run a full build with --force.') - raise SystemExit(1) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") print('Report updated with community labels') -" "INPUT_PATH" +" ``` Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). @@ -537,14 +537,15 @@ If `--obsidian` was given: - If `--obsidian-dir ` was also given, pass it via `--dir`. Otherwise defaults to `graphify-out/obsidian`. ```bash -$(cat graphify-out/.graphify_python) -m graphify export obsidian -# Add `--dir ~/vaults/my-project` to use a custom directory. +graphify export obsidian +# or with custom dir: graphify export obsidian --dir ~/vaults/my-project ``` Generate the HTML graph (always, unless `--no-viz`): ```bash -$(cat graphify-out/.graphify_python) -m graphify export html # auto-aggregates to community view if graph > 5000 nodes +graphify export html # auto-aggregates to community view if graph > 5000 nodes +# or: graphify export html --no-viz ``` ### Steps 6b-8 - Wiki, Neo4j, FalkorDB, SVG, GraphML, MCP, benchmark (only on their flags) @@ -557,7 +558,7 @@ These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, ` ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from pathlib import Path from datetime import datetime, timezone from graphify.detect import save_manifest @@ -580,7 +581,7 @@ extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encod # types are gated on output. from graphify.cli import _stamped_manifest_files _corpus = detect.get('all_files') or detect['files'] -_manifest_files = _stamped_manifest_files(_corpus, extract, Path(sys.argv[1])) +_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) # Files dispatched this run (the changed subset) but NOT stamped above still carry # a stale semantic_hash from a prior run; clear it so detect_incremental re-queues # them instead of reading them as unchanged (#1948). @@ -592,7 +593,7 @@ _cleared = _dispatched - _stamped # files newly excluded since last run are dropped rather than masquerading as # deletions; untouched files' prior rows are still preserved (#1908). _scan = {f for fl in _corpus.values() for f in fl} -save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) # Update cumulative cost tracker input_tok = extract.get('input_tokens', 0) @@ -616,10 +617,10 @@ cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\" print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" "INPUT_PATH" +" rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null -rm -f graphify-out/needs_update 2>/dev/null || true +rm -f graphify-out/.needs_update 2>/dev/null || true ``` Replace INPUT_PATH with the actual path (same value used in Steps 4-5) so the manifest is relativized to the scan root. @@ -684,7 +685,7 @@ Both are non-default subcommands. `--update` re-extracts only new or changed fil When `graphify-out/graph.json` already exists and the user asks a question about the corpus, answer from the graph rather than rebuilding it: ```bash -$(cat graphify-out/.graphify_python) -m graphify query "" +graphify query "" ``` Before traversal, expand the question against the graph's own vocabulary so a wording mismatch does not collapse the answer to noise. If the `graphify query` CLI is unavailable, fall back to an inline NetworkX traversal of `graphify-out/graph.json`. Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. For that vocab-expansion step, the BFS/DFS traversal modes, the `--budget` cap, the NetworkX fallback, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`. diff --git a/tools/skillgen/expected/graphify__skill-codex.md b/tools/skillgen/expected/graphify__skill-codex.md index fbd068db2b..af3f723c78 100644 --- a/tools/skillgen/expected/graphify__skill-codex.md +++ b/tools/skillgen/expected/graphify__skill-codex.md @@ -97,7 +97,7 @@ fi mkdir -p graphify-out "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" # Save scan root so `graphify update` (no args) knows where to look next time -echo "$(cd "INPUT_PATH" && pwd)" > graphify-out/.graphify_root +echo "$(cd INPUT_PATH && pwd)" > graphify-out/.graphify_root ``` If the import succeeds, print nothing and move straight to Step 2. @@ -108,15 +108,15 @@ If the import succeeds, print nothing and move straight to Step 2. ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from graphify.detect import detect from pathlib import Path -result = detect(Path(sys.argv[1])) +result = detect(Path('INPUT_PATH')) # Write the sidecar from Python, not a shell redirect, so the same block renders # on PowerShell hosts without console-encoding drift (#2528). Path('graphify-out/.graphify_detect.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") print(f'Detected {result[\"total_files\"]} files') -" "INPUT_PATH" +" ``` Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: @@ -176,6 +176,7 @@ $(cat graphify-out/.graphify_python) -c " import sys, json from graphify.extract import collect_files, extract from pathlib import Path +import json code_files = [] detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) @@ -183,13 +184,13 @@ for f in detect.get('files', {}).get('code', []): code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) if code_files: - result = extract(code_files, cache_root=Path(sys.argv[1])) + result = extract(code_files, cache_root=Path('INPUT_PATH')) Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') else: Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") print('No code files - skipping AST extraction') -" "INPUT_PATH" +" ``` #### Part B - Semantic extraction (parallel subagents) @@ -204,7 +205,7 @@ Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],' " ``` -**MANDATORY: You MUST use the subagent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the subagent tool you are doing this wrong.** +**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** Before dispatching subagents, print a timing estimate: - Load `total_words` and file counts from `graphify-out/.graphify_detect.json` @@ -220,7 +221,7 @@ SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from graphify.cache import check_semantic_cache from pathlib import Path @@ -230,7 +231,7 @@ detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encodin # every source file (#1392). Video is transcribed to a document in Step 2.5 first. all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root=sys.argv[1], prompt_file=sys.argv[2]) +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') # Always (re)write the cache file: write hits, else DELETE any leftover from a prior # run so Part C never merges a stale .graphify_cached.json (#1392). @@ -240,7 +241,7 @@ else: Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" "INPUT_PATH" "SPEC_PATH" +" ``` Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. @@ -282,7 +283,7 @@ Wait for all subagents. For each result: If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. -Merge all chunk files into `.graphify_semantic_new.json`. **After each subagent call completes, read the real token counts from the subagent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: +Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: ```bash $(cat graphify-out/.graphify_python) -c " import json, glob @@ -309,15 +310,15 @@ print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens' Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root=sys.argv[1], allowed_source_files=uncached, prompt_file=sys.argv[2]) +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') print(f'Cached {saved} files') -" "INPUT_PATH" "SPEC_PATH" +" ``` Merge cached + new results into `graphify-out/.graphify_semantic.json`: @@ -406,7 +407,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -431,7 +432,7 @@ if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') print('If this shrink is intentional (you deleted files), re-run a full build with --force.') raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, sys.argv[1], suggested_questions=questions) +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") analysis = { 'communities': {str(k): v for k, v in communities.items()}, @@ -442,7 +443,7 @@ analysis = { } Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" "INPUT_PATH" +" ``` If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. @@ -455,12 +456,12 @@ A non-destructive diagnostic on the extraction, before labeling. It surfaces edg ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from pathlib import Path from graphify.diagnostics import diagnose_extraction, format_diagnostic_report extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root=sys.argv[1]) +summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( ('dangling_endpoint_edges', 'dangling-endpoint edges'), @@ -470,7 +471,7 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0)] print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') -" "INPUT_PATH" +" ``` Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules). @@ -496,7 +497,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} @@ -507,7 +508,9 @@ labels = LABELS_DICT # Regenerate questions with real community labels (labels affect question phrasing) questions = suggest_questions(G, communities, labels) -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, sys.argv[1], suggested_questions=questions) +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") +Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") # Re-export so graph.json nodes carry the curated community_name (#2490). # Same extraction as Step 4, so the #479 shrink-guard passes on node count; # if it still refuses, surface the guard message - do not force past it. @@ -515,11 +518,8 @@ wrote = to_json(G, communities, 'graphify-out/graph.json', community_labels=labe if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') print('If this shrink is intentional (you deleted files), re-run a full build with --force.') - raise SystemExit(1) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") print('Report updated with community labels') -" "INPUT_PATH" +" ``` Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). @@ -534,14 +534,15 @@ If `--obsidian` was given: - If `--obsidian-dir ` was also given, pass it via `--dir`. Otherwise defaults to `graphify-out/obsidian`. ```bash -$(cat graphify-out/.graphify_python) -m graphify export obsidian -# Add `--dir ~/vaults/my-project` to use a custom directory. +graphify export obsidian +# or with custom dir: graphify export obsidian --dir ~/vaults/my-project ``` Generate the HTML graph (always, unless `--no-viz`): ```bash -$(cat graphify-out/.graphify_python) -m graphify export html # auto-aggregates to community view if graph > 5000 nodes +graphify export html # auto-aggregates to community view if graph > 5000 nodes +# or: graphify export html --no-viz ``` ### Steps 6b-8 - Wiki, Neo4j, FalkorDB, SVG, GraphML, MCP, benchmark (only on their flags) @@ -554,7 +555,7 @@ These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, ` ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from pathlib import Path from datetime import datetime, timezone from graphify.detect import save_manifest @@ -577,7 +578,7 @@ extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encod # types are gated on output. from graphify.cli import _stamped_manifest_files _corpus = detect.get('all_files') or detect['files'] -_manifest_files = _stamped_manifest_files(_corpus, extract, Path(sys.argv[1])) +_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) # Files dispatched this run (the changed subset) but NOT stamped above still carry # a stale semantic_hash from a prior run; clear it so detect_incremental re-queues # them instead of reading them as unchanged (#1948). @@ -589,7 +590,7 @@ _cleared = _dispatched - _stamped # files newly excluded since last run are dropped rather than masquerading as # deletions; untouched files' prior rows are still preserved (#1908). _scan = {f for fl in _corpus.values() for f in fl} -save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) # Update cumulative cost tracker input_tok = extract.get('input_tokens', 0) @@ -613,10 +614,10 @@ cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\" print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" "INPUT_PATH" +" rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null -rm -f graphify-out/needs_update 2>/dev/null || true +rm -f graphify-out/.needs_update 2>/dev/null || true ``` Replace INPUT_PATH with the actual path (same value used in Steps 4-5) so the manifest is relativized to the scan root. @@ -681,7 +682,7 @@ Both are non-default subcommands. `--update` re-extracts only new or changed fil When `graphify-out/graph.json` already exists and the user asks a question about the corpus, answer from the graph rather than rebuilding it: ```bash -$(cat graphify-out/.graphify_python) -m graphify query "" +graphify query "" ``` Before traversal, expand the question against the graph's own vocabulary so a wording mismatch does not collapse the answer to noise. If the `graphify query` CLI is unavailable, fall back to an inline NetworkX traversal of `graphify-out/graph.json`. Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. For that vocab-expansion step, the BFS/DFS traversal modes, the `--budget` cap, the NetworkX fallback, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`. diff --git a/tools/skillgen/expected/graphify__skill-copilot.md b/tools/skillgen/expected/graphify__skill-copilot.md index 55b77d6828..abd2811d23 100644 --- a/tools/skillgen/expected/graphify__skill-copilot.md +++ b/tools/skillgen/expected/graphify__skill-copilot.md @@ -97,7 +97,7 @@ fi mkdir -p graphify-out "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" # Save scan root so `graphify update` (no args) knows where to look next time -echo "$(cd "INPUT_PATH" && pwd)" > graphify-out/.graphify_root +echo "$(cd INPUT_PATH && pwd)" > graphify-out/.graphify_root ``` If the import succeeds, print nothing and move straight to Step 2. @@ -108,15 +108,15 @@ If the import succeeds, print nothing and move straight to Step 2. ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from graphify.detect import detect from pathlib import Path -result = detect(Path(sys.argv[1])) +result = detect(Path('INPUT_PATH')) # Write the sidecar from Python, not a shell redirect, so the same block renders # on PowerShell hosts without console-encoding drift (#2528). Path('graphify-out/.graphify_detect.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") print(f'Detected {result[\"total_files\"]} files') -" "INPUT_PATH" +" ``` Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: @@ -176,6 +176,7 @@ $(cat graphify-out/.graphify_python) -c " import sys, json from graphify.extract import collect_files, extract from pathlib import Path +import json code_files = [] detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) @@ -183,13 +184,13 @@ for f in detect.get('files', {}).get('code', []): code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) if code_files: - result = extract(code_files, cache_root=Path(sys.argv[1])) + result = extract(code_files, cache_root=Path('INPUT_PATH')) Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') else: Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") print('No code files - skipping AST extraction') -" "INPUT_PATH" +" ``` #### Part B - Semantic extraction (parallel subagents) @@ -204,7 +205,7 @@ Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],' " ``` -**MANDATORY: You MUST use the subagent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the subagent tool you are doing this wrong.** +**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** Before dispatching subagents, print a timing estimate: - Load `total_words` and file counts from `graphify-out/.graphify_detect.json` @@ -220,7 +221,7 @@ SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from graphify.cache import check_semantic_cache from pathlib import Path @@ -230,7 +231,7 @@ detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encodin # every source file (#1392). Video is transcribed to a document in Step 2.5 first. all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root=sys.argv[1], prompt_file=sys.argv[2]) +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') # Always (re)write the cache file: write hits, else DELETE any leftover from a prior # run so Part C never merges a stale .graphify_cached.json (#1392). @@ -240,7 +241,7 @@ else: Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" "INPUT_PATH" "SPEC_PATH" +" ``` Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. @@ -285,7 +286,7 @@ Wait for all subagents. For each result: If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. -Merge all chunk files into `.graphify_semantic_new.json`. **After each subagent call completes, read the real token counts from the subagent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: +Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: ```bash $(cat graphify-out/.graphify_python) -c " import json, glob @@ -312,15 +313,15 @@ print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens' Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root=sys.argv[1], allowed_source_files=uncached, prompt_file=sys.argv[2]) +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') print(f'Cached {saved} files') -" "INPUT_PATH" "SPEC_PATH" +" ``` Merge cached + new results into `graphify-out/.graphify_semantic.json`: @@ -409,7 +410,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -434,7 +435,7 @@ if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') print('If this shrink is intentional (you deleted files), re-run a full build with --force.') raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, sys.argv[1], suggested_questions=questions) +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") analysis = { 'communities': {str(k): v for k, v in communities.items()}, @@ -445,7 +446,7 @@ analysis = { } Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" "INPUT_PATH" +" ``` If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. @@ -458,12 +459,12 @@ A non-destructive diagnostic on the extraction, before labeling. It surfaces edg ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from pathlib import Path from graphify.diagnostics import diagnose_extraction, format_diagnostic_report extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root=sys.argv[1]) +summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( ('dangling_endpoint_edges', 'dangling-endpoint edges'), @@ -473,7 +474,7 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0)] print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') -" "INPUT_PATH" +" ``` Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules). @@ -499,7 +500,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} @@ -510,7 +511,9 @@ labels = LABELS_DICT # Regenerate questions with real community labels (labels affect question phrasing) questions = suggest_questions(G, communities, labels) -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, sys.argv[1], suggested_questions=questions) +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") +Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") # Re-export so graph.json nodes carry the curated community_name (#2490). # Same extraction as Step 4, so the #479 shrink-guard passes on node count; # if it still refuses, surface the guard message - do not force past it. @@ -518,11 +521,8 @@ wrote = to_json(G, communities, 'graphify-out/graph.json', community_labels=labe if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') print('If this shrink is intentional (you deleted files), re-run a full build with --force.') - raise SystemExit(1) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") print('Report updated with community labels') -" "INPUT_PATH" +" ``` Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). @@ -537,14 +537,15 @@ If `--obsidian` was given: - If `--obsidian-dir ` was also given, pass it via `--dir`. Otherwise defaults to `graphify-out/obsidian`. ```bash -$(cat graphify-out/.graphify_python) -m graphify export obsidian -# Add `--dir ~/vaults/my-project` to use a custom directory. +graphify export obsidian +# or with custom dir: graphify export obsidian --dir ~/vaults/my-project ``` Generate the HTML graph (always, unless `--no-viz`): ```bash -$(cat graphify-out/.graphify_python) -m graphify export html # auto-aggregates to community view if graph > 5000 nodes +graphify export html # auto-aggregates to community view if graph > 5000 nodes +# or: graphify export html --no-viz ``` ### Steps 6b-8 - Wiki, Neo4j, FalkorDB, SVG, GraphML, MCP, benchmark (only on their flags) @@ -557,7 +558,7 @@ These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, ` ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from pathlib import Path from datetime import datetime, timezone from graphify.detect import save_manifest @@ -580,7 +581,7 @@ extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encod # types are gated on output. from graphify.cli import _stamped_manifest_files _corpus = detect.get('all_files') or detect['files'] -_manifest_files = _stamped_manifest_files(_corpus, extract, Path(sys.argv[1])) +_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) # Files dispatched this run (the changed subset) but NOT stamped above still carry # a stale semantic_hash from a prior run; clear it so detect_incremental re-queues # them instead of reading them as unchanged (#1948). @@ -592,7 +593,7 @@ _cleared = _dispatched - _stamped # files newly excluded since last run are dropped rather than masquerading as # deletions; untouched files' prior rows are still preserved (#1908). _scan = {f for fl in _corpus.values() for f in fl} -save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) # Update cumulative cost tracker input_tok = extract.get('input_tokens', 0) @@ -616,10 +617,10 @@ cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\" print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" "INPUT_PATH" +" rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null -rm -f graphify-out/needs_update 2>/dev/null || true +rm -f graphify-out/.needs_update 2>/dev/null || true ``` Replace INPUT_PATH with the actual path (same value used in Steps 4-5) so the manifest is relativized to the scan root. @@ -684,7 +685,7 @@ Both are non-default subcommands. `--update` re-extracts only new or changed fil When `graphify-out/graph.json` already exists and the user asks a question about the corpus, answer from the graph rather than rebuilding it: ```bash -$(cat graphify-out/.graphify_python) -m graphify query "" +graphify query "" ``` Before traversal, expand the question against the graph's own vocabulary so a wording mismatch does not collapse the answer to noise. If the `graphify query` CLI is unavailable, fall back to an inline NetworkX traversal of `graphify-out/graph.json`. Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. For that vocab-expansion step, the BFS/DFS traversal modes, the `--budget` cap, the NetworkX fallback, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`. diff --git a/tools/skillgen/expected/graphify__skill-devin.md b/tools/skillgen/expected/graphify__skill-devin.md index 8740ca565a..f9be846cbf 100644 --- a/tools/skillgen/expected/graphify__skill-devin.md +++ b/tools/skillgen/expected/graphify__skill-devin.md @@ -114,12 +114,11 @@ If the import succeeds, print nothing and move straight to Step 2. ```bash $(cat graphify-out/.graphify_python) -c " import json -import sys from graphify.detect import detect from pathlib import Path -result = detect(Path(sys.argv[1])) +result = detect(Path('INPUT_PATH')) print(json.dumps(result)) -" "INPUT_PATH" > graphify-out/.graphify_detect.json +" > graphify-out/.graphify_detect.json ``` Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: @@ -147,11 +146,13 @@ Skip this step entirely if `detect` returned zero `video` files. Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. -**Strategy:** If an analysis file exists from a previous run, read its top god-node labels and write a one-sentence domain hint from them. Otherwise use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` Pass the chosen prompt to Whisper as its initial prompt. No separate API call needed. +**Strategy:** Read the god nodes from the detect output or analysis file. You are already a language model - write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. + +**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` **Step 1 - Write the Whisper prompt yourself.** -Read the top god-node labels from previous analysis when available, then compose a short domain hint sentence. With no previous analysis, use the generic fallback above. +Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: - Labels: `transformer, attention, encoder, decoder` -> `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` - Labels: `kubernetes, deployment, pod, helm` -> `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` @@ -490,7 +491,7 @@ wrote = to_json(G, communities, 'graphify-out/graph.json') if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (fewer nodes than the existing graph). Run a full rebuild to be safe.') raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, sys.argv[1], suggested_questions=questions) +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) Path('graphify-out/GRAPH_REPORT.md').write_text(report) analysis = { @@ -502,7 +503,7 @@ analysis = { } Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2)) print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" "INPUT_PATH" +" ``` If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. @@ -540,18 +541,17 @@ labels = LABELS_DICT # Regenerate questions with real community labels (labels affect question phrasing) questions = suggest_questions(G, communities, labels) -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, sys.argv[1], suggested_questions=questions) +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report) +Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()})) # Re-export so graph.json nodes carry the curated community_name (#2490). # Same extraction as Step 4, so the #479 shrink-guard passes on node count; # if it still refuses, surface the guard message - do not force past it. wrote = to_json(G, communities, 'graphify-out/graph.json', community_labels=labels) if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (fewer nodes than the existing graph). Run a full rebuild to be safe.') - raise SystemExit(1) -Path('graphify-out/GRAPH_REPORT.md').write_text(report) -Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()})) print('Report updated with community labels') -" "INPUT_PATH" +" ``` Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). @@ -796,7 +796,6 @@ Print the output directly in chat. If `total_words <= 5000`, skip silently - the ```bash $(cat graphify-out/.graphify_python) -c " import json -import sys from pathlib import Path from datetime import datetime, timezone from graphify.detect import save_manifest @@ -807,13 +806,13 @@ extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) # Stamp only semantic files that produced output so a failed chunk is re-queued next run, not lost (#2015). from graphify.cli import _stamped_manifest_files _corpus = detect.get('all_files') or detect['files'] -_manifest_files = _stamped_manifest_files(_corpus, extract, Path(sys.argv[1])) +_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) _sem_types = ('document', 'paper', 'image') _dispatched = {f for t, fl in detect['files'].items() if t in _sem_types for f in fl} _stamped = {f for fl in _manifest_files.values() for f in fl} _cleared = _dispatched - _stamped _scan = {f for fl in _corpus.values() for f in fl} -save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) # Update cumulative cost tracker input_tok = extract.get('input_tokens', 0) @@ -837,9 +836,9 @@ cost_path.write_text(json.dumps(cost, indent=2)) print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" "INPUT_PATH" +" rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json graphify-out/.graphify_labels.json graphify-out/.graphify_incremental.json graphify-out/.graphify_transcripts.json graphify-out/.graphify_old.json; find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null -rm -f graphify-out/needs_update 2>/dev/null || true +rm -f graphify-out/.needs_update 2>/dev/null || true ``` Tell the user (omit the obsidian line unless --obsidian was given; omit the wiki line unless --wiki was given): @@ -904,7 +903,7 @@ import sys, json from graphify.detect import detect_incremental, save_manifest from pathlib import Path -result = detect_incremental(Path(sys.argv[1])) +result = detect_incremental(Path('INPUT_PATH')) new_total = result.get('new_total', 0) print(json.dumps(result, indent=2)) Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result)) @@ -916,7 +915,7 @@ if deleted: print(f'{len(deleted)} deleted file(s) to prune.') if new_total > 0: print(f'{new_total} new/changed file(s) to re-extract.') -" "INPUT_PATH" +" ``` If new files exist, first check whether all changed files are code files: @@ -959,13 +958,13 @@ $(cat graphify-out/.graphify_python) -c " import sys, json from graphify.build import build_from_json from graphify.export import to_json -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph import networkx as nx from pathlib import Path # Load existing graph existing_data = json.loads(Path('graphify-out/graph.json').read_text()) -G_existing = load_node_link_graph(existing_data) +G_existing = json_graph.node_link_graph(existing_data, edges='links') # Load new extraction new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) @@ -986,7 +985,7 @@ $(cat graphify-out/.graphify_python) -c " import json from graphify.analyze import graph_diff from graphify.build import build_from_json -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph import networkx as nx from pathlib import Path @@ -995,7 +994,7 @@ new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text() G_new = build_from_json(new_extract, directed=IS_DIRECTED) if old_data: - G_old = load_node_link_graph(old_data) + G_old = json_graph.node_link_graph(old_data, edges='links') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: @@ -1021,12 +1020,12 @@ from graphify.cluster import cluster, score_all from graphify.analyze import god_nodes, surprising_connections from graphify.report import generate from graphify.export import to_json -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph import networkx as nx from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text()) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') detection = {'total_files': 0, 'total_words': 99999, 'needs_graph': True, 'warning': None, 'files': {'code': [], 'document': [], 'paper': []}} @@ -1090,12 +1089,12 @@ Load `graphify-out/graph.json`, then: ```bash $(cat graphify-out/.graphify_python) -c " import sys, json -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph import networkx as nx from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text()) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' @@ -1202,11 +1201,11 @@ if not Path('graphify-out/graph.json').exists(): $(cat graphify-out/.graphify_python) -c " import json, sys import networkx as nx -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text()) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') a_term = 'NODE_A' b_term = 'NODE_B' @@ -1274,11 +1273,11 @@ if not Path('graphify-out/graph.json').exists(): $(cat graphify-out/.graphify_python) -c " import json, sys import networkx as nx -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text()) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') term = 'NODE_NAME' term_lower = term.lower() @@ -1332,7 +1331,7 @@ from graphify.ingest import ingest from pathlib import Path try: - out = ingest(sys.argv[1], Path('./raw'), author=sys.argv[2] or None, contributor=sys.argv[3] or None) + out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') print(f'Saved to {out}') except ValueError as e: print(f'error: {e}', file=sys.stderr) @@ -1340,10 +1339,10 @@ except ValueError as e: except RuntimeError as e: print(f'error: {e}', file=sys.stderr) sys.exit(1) -" "URL" "AUTHOR" "CONTRIBUTOR" +" ``` -Replace `URL` with the actual URL and pass the user's `AUTHOR` or `CONTRIBUTOR` when provided; use an empty string for either omitted value. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. +Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. Supported URL types (auto-detected): - Twitter/X -> fetched via oEmbed, saved as `.md` with tweet text and author @@ -1359,7 +1358,7 @@ Supported URL types (auto-detected): Start a background watcher that monitors a folder and auto-updates the graph when files change. ```bash -python3 -m graphify.watch "INPUT_PATH" --debounce 3 +python3 -m graphify.watch INPUT_PATH --debounce 3 ``` Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: diff --git a/tools/skillgen/expected/graphify__skill-droid.md b/tools/skillgen/expected/graphify__skill-droid.md index b9de11f80c..fd148d485d 100644 --- a/tools/skillgen/expected/graphify__skill-droid.md +++ b/tools/skillgen/expected/graphify__skill-droid.md @@ -97,7 +97,7 @@ fi mkdir -p graphify-out "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" # Save scan root so `graphify update` (no args) knows where to look next time -echo "$(cd "INPUT_PATH" && pwd)" > graphify-out/.graphify_root +echo "$(cd INPUT_PATH && pwd)" > graphify-out/.graphify_root ``` If the import succeeds, print nothing and move straight to Step 2. @@ -108,15 +108,15 @@ If the import succeeds, print nothing and move straight to Step 2. ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from graphify.detect import detect from pathlib import Path -result = detect(Path(sys.argv[1])) +result = detect(Path('INPUT_PATH')) # Write the sidecar from Python, not a shell redirect, so the same block renders # on PowerShell hosts without console-encoding drift (#2528). Path('graphify-out/.graphify_detect.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") print(f'Detected {result[\"total_files\"]} files') -" "INPUT_PATH" +" ``` Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: @@ -176,6 +176,7 @@ $(cat graphify-out/.graphify_python) -c " import sys, json from graphify.extract import collect_files, extract from pathlib import Path +import json code_files = [] detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) @@ -183,13 +184,13 @@ for f in detect.get('files', {}).get('code', []): code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) if code_files: - result = extract(code_files, cache_root=Path(sys.argv[1])) + result = extract(code_files, cache_root=Path('INPUT_PATH')) Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') else: Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") print('No code files - skipping AST extraction') -" "INPUT_PATH" +" ``` #### Part B - Semantic extraction (parallel subagents) @@ -204,7 +205,7 @@ Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],' " ``` -**MANDATORY: You MUST use the subagent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the subagent tool you are doing this wrong.** +**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** Before dispatching subagents, print a timing estimate: - Load `total_words` and file counts from `graphify-out/.graphify_detect.json` @@ -220,7 +221,7 @@ SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from graphify.cache import check_semantic_cache from pathlib import Path @@ -230,7 +231,7 @@ detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encodin # every source file (#1392). Video is transcribed to a document in Step 2.5 first. all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root=sys.argv[1], prompt_file=sys.argv[2]) +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') # Always (re)write the cache file: write hits, else DELETE any leftover from a prior # run so Part C never merges a stale .graphify_cached.json (#1392). @@ -240,7 +241,7 @@ else: Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" "INPUT_PATH" "SPEC_PATH" +" ``` Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. @@ -282,7 +283,7 @@ Wait for all subagents. For each result: If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. -Merge all chunk files into `.graphify_semantic_new.json`. **After each subagent call completes, read the real token counts from the subagent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: +Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: ```bash $(cat graphify-out/.graphify_python) -c " import json, glob @@ -309,15 +310,15 @@ print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens' Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root=sys.argv[1], allowed_source_files=uncached, prompt_file=sys.argv[2]) +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') print(f'Cached {saved} files') -" "INPUT_PATH" "SPEC_PATH" +" ``` Merge cached + new results into `graphify-out/.graphify_semantic.json`: @@ -406,7 +407,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -431,7 +432,7 @@ if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') print('If this shrink is intentional (you deleted files), re-run a full build with --force.') raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, sys.argv[1], suggested_questions=questions) +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") analysis = { 'communities': {str(k): v for k, v in communities.items()}, @@ -442,7 +443,7 @@ analysis = { } Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" "INPUT_PATH" +" ``` If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. @@ -455,12 +456,12 @@ A non-destructive diagnostic on the extraction, before labeling. It surfaces edg ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from pathlib import Path from graphify.diagnostics import diagnose_extraction, format_diagnostic_report extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root=sys.argv[1]) +summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( ('dangling_endpoint_edges', 'dangling-endpoint edges'), @@ -470,7 +471,7 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0)] print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') -" "INPUT_PATH" +" ``` Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules). @@ -496,7 +497,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} @@ -507,7 +508,9 @@ labels = LABELS_DICT # Regenerate questions with real community labels (labels affect question phrasing) questions = suggest_questions(G, communities, labels) -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, sys.argv[1], suggested_questions=questions) +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") +Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") # Re-export so graph.json nodes carry the curated community_name (#2490). # Same extraction as Step 4, so the #479 shrink-guard passes on node count; # if it still refuses, surface the guard message - do not force past it. @@ -515,11 +518,8 @@ wrote = to_json(G, communities, 'graphify-out/graph.json', community_labels=labe if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') print('If this shrink is intentional (you deleted files), re-run a full build with --force.') - raise SystemExit(1) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") print('Report updated with community labels') -" "INPUT_PATH" +" ``` Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). @@ -534,14 +534,15 @@ If `--obsidian` was given: - If `--obsidian-dir ` was also given, pass it via `--dir`. Otherwise defaults to `graphify-out/obsidian`. ```bash -$(cat graphify-out/.graphify_python) -m graphify export obsidian -# Add `--dir ~/vaults/my-project` to use a custom directory. +graphify export obsidian +# or with custom dir: graphify export obsidian --dir ~/vaults/my-project ``` Generate the HTML graph (always, unless `--no-viz`): ```bash -$(cat graphify-out/.graphify_python) -m graphify export html # auto-aggregates to community view if graph > 5000 nodes +graphify export html # auto-aggregates to community view if graph > 5000 nodes +# or: graphify export html --no-viz ``` ### Steps 6b-8 - Wiki, Neo4j, FalkorDB, SVG, GraphML, MCP, benchmark (only on their flags) @@ -554,7 +555,7 @@ These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, ` ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from pathlib import Path from datetime import datetime, timezone from graphify.detect import save_manifest @@ -577,7 +578,7 @@ extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encod # types are gated on output. from graphify.cli import _stamped_manifest_files _corpus = detect.get('all_files') or detect['files'] -_manifest_files = _stamped_manifest_files(_corpus, extract, Path(sys.argv[1])) +_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) # Files dispatched this run (the changed subset) but NOT stamped above still carry # a stale semantic_hash from a prior run; clear it so detect_incremental re-queues # them instead of reading them as unchanged (#1948). @@ -589,7 +590,7 @@ _cleared = _dispatched - _stamped # files newly excluded since last run are dropped rather than masquerading as # deletions; untouched files' prior rows are still preserved (#1908). _scan = {f for fl in _corpus.values() for f in fl} -save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) # Update cumulative cost tracker input_tok = extract.get('input_tokens', 0) @@ -613,10 +614,10 @@ cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\" print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" "INPUT_PATH" +" rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null -rm -f graphify-out/needs_update 2>/dev/null || true +rm -f graphify-out/.needs_update 2>/dev/null || true ``` Replace INPUT_PATH with the actual path (same value used in Steps 4-5) so the manifest is relativized to the scan root. @@ -681,7 +682,7 @@ Both are non-default subcommands. `--update` re-extracts only new or changed fil When `graphify-out/graph.json` already exists and the user asks a question about the corpus, answer from the graph rather than rebuilding it: ```bash -$(cat graphify-out/.graphify_python) -m graphify query "" +graphify query "" ``` Before traversal, expand the question against the graph's own vocabulary so a wording mismatch does not collapse the answer to noise. If the `graphify query` CLI is unavailable, fall back to an inline NetworkX traversal of `graphify-out/graph.json`. Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. For that vocab-expansion step, the BFS/DFS traversal modes, the `--budget` cap, the NetworkX fallback, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`. diff --git a/tools/skillgen/expected/graphify__skill-kilo.md b/tools/skillgen/expected/graphify__skill-kilo.md index ef93d78a4d..3e70b050a4 100644 --- a/tools/skillgen/expected/graphify__skill-kilo.md +++ b/tools/skillgen/expected/graphify__skill-kilo.md @@ -97,7 +97,7 @@ fi mkdir -p graphify-out "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" # Save scan root so `graphify update` (no args) knows where to look next time -echo "$(cd "INPUT_PATH" && pwd)" > graphify-out/.graphify_root +echo "$(cd INPUT_PATH && pwd)" > graphify-out/.graphify_root ``` If the import succeeds, print nothing and move straight to Step 2. @@ -108,15 +108,15 @@ If the import succeeds, print nothing and move straight to Step 2. ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from graphify.detect import detect from pathlib import Path -result = detect(Path(sys.argv[1])) +result = detect(Path('INPUT_PATH')) # Write the sidecar from Python, not a shell redirect, so the same block renders # on PowerShell hosts without console-encoding drift (#2528). Path('graphify-out/.graphify_detect.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") print(f'Detected {result[\"total_files\"]} files') -" "INPUT_PATH" +" ``` Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: @@ -176,6 +176,7 @@ $(cat graphify-out/.graphify_python) -c " import sys, json from graphify.extract import collect_files, extract from pathlib import Path +import json code_files = [] detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) @@ -183,13 +184,13 @@ for f in detect.get('files', {}).get('code', []): code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) if code_files: - result = extract(code_files, cache_root=Path(sys.argv[1])) + result = extract(code_files, cache_root=Path('INPUT_PATH')) Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') else: Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") print('No code files - skipping AST extraction') -" "INPUT_PATH" +" ``` #### Part B - Semantic extraction (parallel subagents) @@ -204,7 +205,7 @@ Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],' " ``` -**MANDATORY: You MUST use the subagent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the subagent tool you are doing this wrong.** +**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** Before dispatching subagents, print a timing estimate: - Load `total_words` and file counts from `graphify-out/.graphify_detect.json` @@ -220,7 +221,7 @@ SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from graphify.cache import check_semantic_cache from pathlib import Path @@ -230,7 +231,7 @@ detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encodin # every source file (#1392). Video is transcribed to a document in Step 2.5 first. all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root=sys.argv[1], prompt_file=sys.argv[2]) +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') # Always (re)write the cache file: write hits, else DELETE any leftover from a prior # run so Part C never merges a stale .graphify_cached.json (#1392). @@ -240,7 +241,7 @@ else: Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" "INPUT_PATH" "SPEC_PATH" +" ``` Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. @@ -285,7 +286,7 @@ Wait for all subagents. For each result: If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. -Merge all chunk files into `.graphify_semantic_new.json`. **After each subagent call completes, read the real token counts from the subagent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: +Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: ```bash $(cat graphify-out/.graphify_python) -c " import json, glob @@ -312,15 +313,15 @@ print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens' Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root=sys.argv[1], allowed_source_files=uncached, prompt_file=sys.argv[2]) +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') print(f'Cached {saved} files') -" "INPUT_PATH" "SPEC_PATH" +" ``` Merge cached + new results into `graphify-out/.graphify_semantic.json`: @@ -409,7 +410,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -434,7 +435,7 @@ if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') print('If this shrink is intentional (you deleted files), re-run a full build with --force.') raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, sys.argv[1], suggested_questions=questions) +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") analysis = { 'communities': {str(k): v for k, v in communities.items()}, @@ -445,7 +446,7 @@ analysis = { } Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" "INPUT_PATH" +" ``` If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. @@ -458,12 +459,12 @@ A non-destructive diagnostic on the extraction, before labeling. It surfaces edg ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from pathlib import Path from graphify.diagnostics import diagnose_extraction, format_diagnostic_report extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root=sys.argv[1]) +summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( ('dangling_endpoint_edges', 'dangling-endpoint edges'), @@ -473,7 +474,7 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0)] print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') -" "INPUT_PATH" +" ``` Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules). @@ -499,7 +500,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} @@ -510,7 +511,9 @@ labels = LABELS_DICT # Regenerate questions with real community labels (labels affect question phrasing) questions = suggest_questions(G, communities, labels) -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, sys.argv[1], suggested_questions=questions) +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") +Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") # Re-export so graph.json nodes carry the curated community_name (#2490). # Same extraction as Step 4, so the #479 shrink-guard passes on node count; # if it still refuses, surface the guard message - do not force past it. @@ -518,11 +521,8 @@ wrote = to_json(G, communities, 'graphify-out/graph.json', community_labels=labe if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') print('If this shrink is intentional (you deleted files), re-run a full build with --force.') - raise SystemExit(1) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") print('Report updated with community labels') -" "INPUT_PATH" +" ``` Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). @@ -537,14 +537,15 @@ If `--obsidian` was given: - If `--obsidian-dir ` was also given, pass it via `--dir`. Otherwise defaults to `graphify-out/obsidian`. ```bash -$(cat graphify-out/.graphify_python) -m graphify export obsidian -# Add `--dir ~/vaults/my-project` to use a custom directory. +graphify export obsidian +# or with custom dir: graphify export obsidian --dir ~/vaults/my-project ``` Generate the HTML graph (always, unless `--no-viz`): ```bash -$(cat graphify-out/.graphify_python) -m graphify export html # auto-aggregates to community view if graph > 5000 nodes +graphify export html # auto-aggregates to community view if graph > 5000 nodes +# or: graphify export html --no-viz ``` ### Steps 6b-8 - Wiki, Neo4j, FalkorDB, SVG, GraphML, MCP, benchmark (only on their flags) @@ -557,7 +558,7 @@ These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, ` ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from pathlib import Path from datetime import datetime, timezone from graphify.detect import save_manifest @@ -580,7 +581,7 @@ extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encod # types are gated on output. from graphify.cli import _stamped_manifest_files _corpus = detect.get('all_files') or detect['files'] -_manifest_files = _stamped_manifest_files(_corpus, extract, Path(sys.argv[1])) +_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) # Files dispatched this run (the changed subset) but NOT stamped above still carry # a stale semantic_hash from a prior run; clear it so detect_incremental re-queues # them instead of reading them as unchanged (#1948). @@ -592,7 +593,7 @@ _cleared = _dispatched - _stamped # files newly excluded since last run are dropped rather than masquerading as # deletions; untouched files' prior rows are still preserved (#1908). _scan = {f for fl in _corpus.values() for f in fl} -save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) # Update cumulative cost tracker input_tok = extract.get('input_tokens', 0) @@ -616,10 +617,10 @@ cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\" print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" "INPUT_PATH" +" rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null -rm -f graphify-out/needs_update 2>/dev/null || true +rm -f graphify-out/.needs_update 2>/dev/null || true ``` Replace INPUT_PATH with the actual path (same value used in Steps 4-5) so the manifest is relativized to the scan root. @@ -684,7 +685,7 @@ Both are non-default subcommands. `--update` re-extracts only new or changed fil When `graphify-out/graph.json` already exists and the user asks a question about the corpus, answer from the graph rather than rebuilding it: ```bash -$(cat graphify-out/.graphify_python) -m graphify query "" +graphify query "" ``` Before traversal, expand the question against the graph's own vocabulary so a wording mismatch does not collapse the answer to noise. If the `graphify query` CLI is unavailable, fall back to an inline NetworkX traversal of `graphify-out/graph.json`. Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. For that vocab-expansion step, the BFS/DFS traversal modes, the `--budget` cap, the NetworkX fallback, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`. diff --git a/tools/skillgen/expected/graphify__skill-kiro.md b/tools/skillgen/expected/graphify__skill-kiro.md index 55b77d6828..abd2811d23 100644 --- a/tools/skillgen/expected/graphify__skill-kiro.md +++ b/tools/skillgen/expected/graphify__skill-kiro.md @@ -97,7 +97,7 @@ fi mkdir -p graphify-out "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" # Save scan root so `graphify update` (no args) knows where to look next time -echo "$(cd "INPUT_PATH" && pwd)" > graphify-out/.graphify_root +echo "$(cd INPUT_PATH && pwd)" > graphify-out/.graphify_root ``` If the import succeeds, print nothing and move straight to Step 2. @@ -108,15 +108,15 @@ If the import succeeds, print nothing and move straight to Step 2. ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from graphify.detect import detect from pathlib import Path -result = detect(Path(sys.argv[1])) +result = detect(Path('INPUT_PATH')) # Write the sidecar from Python, not a shell redirect, so the same block renders # on PowerShell hosts without console-encoding drift (#2528). Path('graphify-out/.graphify_detect.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") print(f'Detected {result[\"total_files\"]} files') -" "INPUT_PATH" +" ``` Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: @@ -176,6 +176,7 @@ $(cat graphify-out/.graphify_python) -c " import sys, json from graphify.extract import collect_files, extract from pathlib import Path +import json code_files = [] detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) @@ -183,13 +184,13 @@ for f in detect.get('files', {}).get('code', []): code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) if code_files: - result = extract(code_files, cache_root=Path(sys.argv[1])) + result = extract(code_files, cache_root=Path('INPUT_PATH')) Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') else: Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") print('No code files - skipping AST extraction') -" "INPUT_PATH" +" ``` #### Part B - Semantic extraction (parallel subagents) @@ -204,7 +205,7 @@ Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],' " ``` -**MANDATORY: You MUST use the subagent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the subagent tool you are doing this wrong.** +**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** Before dispatching subagents, print a timing estimate: - Load `total_words` and file counts from `graphify-out/.graphify_detect.json` @@ -220,7 +221,7 @@ SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from graphify.cache import check_semantic_cache from pathlib import Path @@ -230,7 +231,7 @@ detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encodin # every source file (#1392). Video is transcribed to a document in Step 2.5 first. all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root=sys.argv[1], prompt_file=sys.argv[2]) +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') # Always (re)write the cache file: write hits, else DELETE any leftover from a prior # run so Part C never merges a stale .graphify_cached.json (#1392). @@ -240,7 +241,7 @@ else: Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" "INPUT_PATH" "SPEC_PATH" +" ``` Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. @@ -285,7 +286,7 @@ Wait for all subagents. For each result: If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. -Merge all chunk files into `.graphify_semantic_new.json`. **After each subagent call completes, read the real token counts from the subagent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: +Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: ```bash $(cat graphify-out/.graphify_python) -c " import json, glob @@ -312,15 +313,15 @@ print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens' Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root=sys.argv[1], allowed_source_files=uncached, prompt_file=sys.argv[2]) +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') print(f'Cached {saved} files') -" "INPUT_PATH" "SPEC_PATH" +" ``` Merge cached + new results into `graphify-out/.graphify_semantic.json`: @@ -409,7 +410,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -434,7 +435,7 @@ if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') print('If this shrink is intentional (you deleted files), re-run a full build with --force.') raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, sys.argv[1], suggested_questions=questions) +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") analysis = { 'communities': {str(k): v for k, v in communities.items()}, @@ -445,7 +446,7 @@ analysis = { } Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" "INPUT_PATH" +" ``` If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. @@ -458,12 +459,12 @@ A non-destructive diagnostic on the extraction, before labeling. It surfaces edg ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from pathlib import Path from graphify.diagnostics import diagnose_extraction, format_diagnostic_report extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root=sys.argv[1]) +summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( ('dangling_endpoint_edges', 'dangling-endpoint edges'), @@ -473,7 +474,7 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0)] print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') -" "INPUT_PATH" +" ``` Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules). @@ -499,7 +500,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} @@ -510,7 +511,9 @@ labels = LABELS_DICT # Regenerate questions with real community labels (labels affect question phrasing) questions = suggest_questions(G, communities, labels) -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, sys.argv[1], suggested_questions=questions) +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") +Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") # Re-export so graph.json nodes carry the curated community_name (#2490). # Same extraction as Step 4, so the #479 shrink-guard passes on node count; # if it still refuses, surface the guard message - do not force past it. @@ -518,11 +521,8 @@ wrote = to_json(G, communities, 'graphify-out/graph.json', community_labels=labe if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') print('If this shrink is intentional (you deleted files), re-run a full build with --force.') - raise SystemExit(1) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") print('Report updated with community labels') -" "INPUT_PATH" +" ``` Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). @@ -537,14 +537,15 @@ If `--obsidian` was given: - If `--obsidian-dir ` was also given, pass it via `--dir`. Otherwise defaults to `graphify-out/obsidian`. ```bash -$(cat graphify-out/.graphify_python) -m graphify export obsidian -# Add `--dir ~/vaults/my-project` to use a custom directory. +graphify export obsidian +# or with custom dir: graphify export obsidian --dir ~/vaults/my-project ``` Generate the HTML graph (always, unless `--no-viz`): ```bash -$(cat graphify-out/.graphify_python) -m graphify export html # auto-aggregates to community view if graph > 5000 nodes +graphify export html # auto-aggregates to community view if graph > 5000 nodes +# or: graphify export html --no-viz ``` ### Steps 6b-8 - Wiki, Neo4j, FalkorDB, SVG, GraphML, MCP, benchmark (only on their flags) @@ -557,7 +558,7 @@ These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, ` ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from pathlib import Path from datetime import datetime, timezone from graphify.detect import save_manifest @@ -580,7 +581,7 @@ extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encod # types are gated on output. from graphify.cli import _stamped_manifest_files _corpus = detect.get('all_files') or detect['files'] -_manifest_files = _stamped_manifest_files(_corpus, extract, Path(sys.argv[1])) +_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) # Files dispatched this run (the changed subset) but NOT stamped above still carry # a stale semantic_hash from a prior run; clear it so detect_incremental re-queues # them instead of reading them as unchanged (#1948). @@ -592,7 +593,7 @@ _cleared = _dispatched - _stamped # files newly excluded since last run are dropped rather than masquerading as # deletions; untouched files' prior rows are still preserved (#1908). _scan = {f for fl in _corpus.values() for f in fl} -save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) # Update cumulative cost tracker input_tok = extract.get('input_tokens', 0) @@ -616,10 +617,10 @@ cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\" print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" "INPUT_PATH" +" rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null -rm -f graphify-out/needs_update 2>/dev/null || true +rm -f graphify-out/.needs_update 2>/dev/null || true ``` Replace INPUT_PATH with the actual path (same value used in Steps 4-5) so the manifest is relativized to the scan root. @@ -684,7 +685,7 @@ Both are non-default subcommands. `--update` re-extracts only new or changed fil When `graphify-out/graph.json` already exists and the user asks a question about the corpus, answer from the graph rather than rebuilding it: ```bash -$(cat graphify-out/.graphify_python) -m graphify query "" +graphify query "" ``` Before traversal, expand the question against the graph's own vocabulary so a wording mismatch does not collapse the answer to noise. If the `graphify query` CLI is unavailable, fall back to an inline NetworkX traversal of `graphify-out/graph.json`. Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. For that vocab-expansion step, the BFS/DFS traversal modes, the `--budget` cap, the NetworkX fallback, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`. diff --git a/tools/skillgen/expected/graphify__skill-opencode.md b/tools/skillgen/expected/graphify__skill-opencode.md index f1afcb1aa3..91ced60675 100644 --- a/tools/skillgen/expected/graphify__skill-opencode.md +++ b/tools/skillgen/expected/graphify__skill-opencode.md @@ -97,7 +97,7 @@ fi mkdir -p graphify-out "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" # Save scan root so `graphify update` (no args) knows where to look next time -echo "$(cd "INPUT_PATH" && pwd)" > graphify-out/.graphify_root +echo "$(cd INPUT_PATH && pwd)" > graphify-out/.graphify_root ``` If the import succeeds, print nothing and move straight to Step 2. @@ -108,15 +108,15 @@ If the import succeeds, print nothing and move straight to Step 2. ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from graphify.detect import detect from pathlib import Path -result = detect(Path(sys.argv[1])) +result = detect(Path('INPUT_PATH')) # Write the sidecar from Python, not a shell redirect, so the same block renders # on PowerShell hosts without console-encoding drift (#2528). Path('graphify-out/.graphify_detect.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") print(f'Detected {result[\"total_files\"]} files') -" "INPUT_PATH" +" ``` Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: @@ -176,6 +176,7 @@ $(cat graphify-out/.graphify_python) -c " import sys, json from graphify.extract import collect_files, extract from pathlib import Path +import json code_files = [] detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) @@ -183,13 +184,13 @@ for f in detect.get('files', {}).get('code', []): code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) if code_files: - result = extract(code_files, cache_root=Path(sys.argv[1])) + result = extract(code_files, cache_root=Path('INPUT_PATH')) Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') else: Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") print('No code files - skipping AST extraction') -" "INPUT_PATH" +" ``` #### Part B - Semantic extraction (parallel subagents) @@ -204,7 +205,7 @@ Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],' " ``` -**MANDATORY: You MUST use the subagent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the subagent tool you are doing this wrong.** +**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** Before dispatching subagents, print a timing estimate: - Load `total_words` and file counts from `graphify-out/.graphify_detect.json` @@ -220,7 +221,7 @@ SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from graphify.cache import check_semantic_cache from pathlib import Path @@ -230,7 +231,7 @@ detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encodin # every source file (#1392). Video is transcribed to a document in Step 2.5 first. all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root=sys.argv[1], prompt_file=sys.argv[2]) +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') # Always (re)write the cache file: write hits, else DELETE any leftover from a prior # run so Part C never merges a stale .graphify_cached.json (#1392). @@ -240,7 +241,7 @@ else: Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" "INPUT_PATH" "SPEC_PATH" +" ``` Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. @@ -277,7 +278,7 @@ Wait for all subagents. For each result: If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. -Merge all chunk files into `.graphify_semantic_new.json`. **After each subagent call completes, read the real token counts from the subagent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: +Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: ```bash $(cat graphify-out/.graphify_python) -c " import json, glob @@ -304,15 +305,15 @@ print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens' Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root=sys.argv[1], allowed_source_files=uncached, prompt_file=sys.argv[2]) +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') print(f'Cached {saved} files') -" "INPUT_PATH" "SPEC_PATH" +" ``` Merge cached + new results into `graphify-out/.graphify_semantic.json`: @@ -401,7 +402,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -426,7 +427,7 @@ if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') print('If this shrink is intentional (you deleted files), re-run a full build with --force.') raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, sys.argv[1], suggested_questions=questions) +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") analysis = { 'communities': {str(k): v for k, v in communities.items()}, @@ -437,7 +438,7 @@ analysis = { } Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" "INPUT_PATH" +" ``` If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. @@ -450,12 +451,12 @@ A non-destructive diagnostic on the extraction, before labeling. It surfaces edg ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from pathlib import Path from graphify.diagnostics import diagnose_extraction, format_diagnostic_report extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root=sys.argv[1]) +summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( ('dangling_endpoint_edges', 'dangling-endpoint edges'), @@ -465,7 +466,7 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0)] print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') -" "INPUT_PATH" +" ``` Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules). @@ -491,7 +492,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} @@ -502,7 +503,9 @@ labels = LABELS_DICT # Regenerate questions with real community labels (labels affect question phrasing) questions = suggest_questions(G, communities, labels) -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, sys.argv[1], suggested_questions=questions) +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") +Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") # Re-export so graph.json nodes carry the curated community_name (#2490). # Same extraction as Step 4, so the #479 shrink-guard passes on node count; # if it still refuses, surface the guard message - do not force past it. @@ -510,11 +513,8 @@ wrote = to_json(G, communities, 'graphify-out/graph.json', community_labels=labe if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') print('If this shrink is intentional (you deleted files), re-run a full build with --force.') - raise SystemExit(1) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") print('Report updated with community labels') -" "INPUT_PATH" +" ``` Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). @@ -529,14 +529,15 @@ If `--obsidian` was given: - If `--obsidian-dir ` was also given, pass it via `--dir`. Otherwise defaults to `graphify-out/obsidian`. ```bash -$(cat graphify-out/.graphify_python) -m graphify export obsidian -# Add `--dir ~/vaults/my-project` to use a custom directory. +graphify export obsidian +# or with custom dir: graphify export obsidian --dir ~/vaults/my-project ``` Generate the HTML graph (always, unless `--no-viz`): ```bash -$(cat graphify-out/.graphify_python) -m graphify export html # auto-aggregates to community view if graph > 5000 nodes +graphify export html # auto-aggregates to community view if graph > 5000 nodes +# or: graphify export html --no-viz ``` ### Steps 6b-8 - Wiki, Neo4j, FalkorDB, SVG, GraphML, MCP, benchmark (only on their flags) @@ -549,7 +550,7 @@ These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, ` ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from pathlib import Path from datetime import datetime, timezone from graphify.detect import save_manifest @@ -572,7 +573,7 @@ extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encod # types are gated on output. from graphify.cli import _stamped_manifest_files _corpus = detect.get('all_files') or detect['files'] -_manifest_files = _stamped_manifest_files(_corpus, extract, Path(sys.argv[1])) +_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) # Files dispatched this run (the changed subset) but NOT stamped above still carry # a stale semantic_hash from a prior run; clear it so detect_incremental re-queues # them instead of reading them as unchanged (#1948). @@ -584,7 +585,7 @@ _cleared = _dispatched - _stamped # files newly excluded since last run are dropped rather than masquerading as # deletions; untouched files' prior rows are still preserved (#1908). _scan = {f for fl in _corpus.values() for f in fl} -save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) # Update cumulative cost tracker input_tok = extract.get('input_tokens', 0) @@ -608,10 +609,10 @@ cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\" print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" "INPUT_PATH" +" rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null -rm -f graphify-out/needs_update 2>/dev/null || true +rm -f graphify-out/.needs_update 2>/dev/null || true ``` Replace INPUT_PATH with the actual path (same value used in Steps 4-5) so the manifest is relativized to the scan root. @@ -676,7 +677,7 @@ Both are non-default subcommands. `--update` re-extracts only new or changed fil When `graphify-out/graph.json` already exists and the user asks a question about the corpus, answer from the graph rather than rebuilding it: ```bash -$(cat graphify-out/.graphify_python) -m graphify query "" +graphify query "" ``` Before traversal, expand the question against the graph's own vocabulary so a wording mismatch does not collapse the answer to noise. If the `graphify query` CLI is unavailable, fall back to an inline NetworkX traversal of `graphify-out/graph.json`. Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. For that vocab-expansion step, the BFS/DFS traversal modes, the `--budget` cap, the NetworkX fallback, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`. diff --git a/tools/skillgen/expected/graphify__skill-pi.md b/tools/skillgen/expected/graphify__skill-pi.md index 55b77d6828..abd2811d23 100644 --- a/tools/skillgen/expected/graphify__skill-pi.md +++ b/tools/skillgen/expected/graphify__skill-pi.md @@ -97,7 +97,7 @@ fi mkdir -p graphify-out "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" # Save scan root so `graphify update` (no args) knows where to look next time -echo "$(cd "INPUT_PATH" && pwd)" > graphify-out/.graphify_root +echo "$(cd INPUT_PATH && pwd)" > graphify-out/.graphify_root ``` If the import succeeds, print nothing and move straight to Step 2. @@ -108,15 +108,15 @@ If the import succeeds, print nothing and move straight to Step 2. ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from graphify.detect import detect from pathlib import Path -result = detect(Path(sys.argv[1])) +result = detect(Path('INPUT_PATH')) # Write the sidecar from Python, not a shell redirect, so the same block renders # on PowerShell hosts without console-encoding drift (#2528). Path('graphify-out/.graphify_detect.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") print(f'Detected {result[\"total_files\"]} files') -" "INPUT_PATH" +" ``` Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: @@ -176,6 +176,7 @@ $(cat graphify-out/.graphify_python) -c " import sys, json from graphify.extract import collect_files, extract from pathlib import Path +import json code_files = [] detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) @@ -183,13 +184,13 @@ for f in detect.get('files', {}).get('code', []): code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) if code_files: - result = extract(code_files, cache_root=Path(sys.argv[1])) + result = extract(code_files, cache_root=Path('INPUT_PATH')) Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') else: Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") print('No code files - skipping AST extraction') -" "INPUT_PATH" +" ``` #### Part B - Semantic extraction (parallel subagents) @@ -204,7 +205,7 @@ Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],' " ``` -**MANDATORY: You MUST use the subagent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the subagent tool you are doing this wrong.** +**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** Before dispatching subagents, print a timing estimate: - Load `total_words` and file counts from `graphify-out/.graphify_detect.json` @@ -220,7 +221,7 @@ SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from graphify.cache import check_semantic_cache from pathlib import Path @@ -230,7 +231,7 @@ detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encodin # every source file (#1392). Video is transcribed to a document in Step 2.5 first. all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root=sys.argv[1], prompt_file=sys.argv[2]) +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') # Always (re)write the cache file: write hits, else DELETE any leftover from a prior # run so Part C never merges a stale .graphify_cached.json (#1392). @@ -240,7 +241,7 @@ else: Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" "INPUT_PATH" "SPEC_PATH" +" ``` Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. @@ -285,7 +286,7 @@ Wait for all subagents. For each result: If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. -Merge all chunk files into `.graphify_semantic_new.json`. **After each subagent call completes, read the real token counts from the subagent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: +Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: ```bash $(cat graphify-out/.graphify_python) -c " import json, glob @@ -312,15 +313,15 @@ print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens' Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root=sys.argv[1], allowed_source_files=uncached, prompt_file=sys.argv[2]) +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') print(f'Cached {saved} files') -" "INPUT_PATH" "SPEC_PATH" +" ``` Merge cached + new results into `graphify-out/.graphify_semantic.json`: @@ -409,7 +410,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -434,7 +435,7 @@ if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') print('If this shrink is intentional (you deleted files), re-run a full build with --force.') raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, sys.argv[1], suggested_questions=questions) +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") analysis = { 'communities': {str(k): v for k, v in communities.items()}, @@ -445,7 +446,7 @@ analysis = { } Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" "INPUT_PATH" +" ``` If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. @@ -458,12 +459,12 @@ A non-destructive diagnostic on the extraction, before labeling. It surfaces edg ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from pathlib import Path from graphify.diagnostics import diagnose_extraction, format_diagnostic_report extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root=sys.argv[1]) +summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( ('dangling_endpoint_edges', 'dangling-endpoint edges'), @@ -473,7 +474,7 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0)] print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') -" "INPUT_PATH" +" ``` Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules). @@ -499,7 +500,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} @@ -510,7 +511,9 @@ labels = LABELS_DICT # Regenerate questions with real community labels (labels affect question phrasing) questions = suggest_questions(G, communities, labels) -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, sys.argv[1], suggested_questions=questions) +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") +Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") # Re-export so graph.json nodes carry the curated community_name (#2490). # Same extraction as Step 4, so the #479 shrink-guard passes on node count; # if it still refuses, surface the guard message - do not force past it. @@ -518,11 +521,8 @@ wrote = to_json(G, communities, 'graphify-out/graph.json', community_labels=labe if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') print('If this shrink is intentional (you deleted files), re-run a full build with --force.') - raise SystemExit(1) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") print('Report updated with community labels') -" "INPUT_PATH" +" ``` Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). @@ -537,14 +537,15 @@ If `--obsidian` was given: - If `--obsidian-dir ` was also given, pass it via `--dir`. Otherwise defaults to `graphify-out/obsidian`. ```bash -$(cat graphify-out/.graphify_python) -m graphify export obsidian -# Add `--dir ~/vaults/my-project` to use a custom directory. +graphify export obsidian +# or with custom dir: graphify export obsidian --dir ~/vaults/my-project ``` Generate the HTML graph (always, unless `--no-viz`): ```bash -$(cat graphify-out/.graphify_python) -m graphify export html # auto-aggregates to community view if graph > 5000 nodes +graphify export html # auto-aggregates to community view if graph > 5000 nodes +# or: graphify export html --no-viz ``` ### Steps 6b-8 - Wiki, Neo4j, FalkorDB, SVG, GraphML, MCP, benchmark (only on their flags) @@ -557,7 +558,7 @@ These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, ` ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from pathlib import Path from datetime import datetime, timezone from graphify.detect import save_manifest @@ -580,7 +581,7 @@ extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encod # types are gated on output. from graphify.cli import _stamped_manifest_files _corpus = detect.get('all_files') or detect['files'] -_manifest_files = _stamped_manifest_files(_corpus, extract, Path(sys.argv[1])) +_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) # Files dispatched this run (the changed subset) but NOT stamped above still carry # a stale semantic_hash from a prior run; clear it so detect_incremental re-queues # them instead of reading them as unchanged (#1948). @@ -592,7 +593,7 @@ _cleared = _dispatched - _stamped # files newly excluded since last run are dropped rather than masquerading as # deletions; untouched files' prior rows are still preserved (#1908). _scan = {f for fl in _corpus.values() for f in fl} -save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) # Update cumulative cost tracker input_tok = extract.get('input_tokens', 0) @@ -616,10 +617,10 @@ cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\" print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" "INPUT_PATH" +" rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null -rm -f graphify-out/needs_update 2>/dev/null || true +rm -f graphify-out/.needs_update 2>/dev/null || true ``` Replace INPUT_PATH with the actual path (same value used in Steps 4-5) so the manifest is relativized to the scan root. @@ -684,7 +685,7 @@ Both are non-default subcommands. `--update` re-extracts only new or changed fil When `graphify-out/graph.json` already exists and the user asks a question about the corpus, answer from the graph rather than rebuilding it: ```bash -$(cat graphify-out/.graphify_python) -m graphify query "" +graphify query "" ``` Before traversal, expand the question against the graph's own vocabulary so a wording mismatch does not collapse the answer to noise. If the `graphify query` CLI is unavailable, fall back to an inline NetworkX traversal of `graphify-out/graph.json`. Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. For that vocab-expansion step, the BFS/DFS traversal modes, the `--budget` cap, the NetworkX fallback, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`. diff --git a/tools/skillgen/expected/graphify__skill-trae.md b/tools/skillgen/expected/graphify__skill-trae.md index 052dc548af..050667bc20 100644 --- a/tools/skillgen/expected/graphify__skill-trae.md +++ b/tools/skillgen/expected/graphify__skill-trae.md @@ -97,7 +97,7 @@ fi mkdir -p graphify-out "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" # Save scan root so `graphify update` (no args) knows where to look next time -echo "$(cd "INPUT_PATH" && pwd)" > graphify-out/.graphify_root +echo "$(cd INPUT_PATH && pwd)" > graphify-out/.graphify_root ``` If the import succeeds, print nothing and move straight to Step 2. @@ -108,15 +108,15 @@ If the import succeeds, print nothing and move straight to Step 2. ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from graphify.detect import detect from pathlib import Path -result = detect(Path(sys.argv[1])) +result = detect(Path('INPUT_PATH')) # Write the sidecar from Python, not a shell redirect, so the same block renders # on PowerShell hosts without console-encoding drift (#2528). Path('graphify-out/.graphify_detect.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") print(f'Detected {result[\"total_files\"]} files') -" "INPUT_PATH" +" ``` Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: @@ -176,6 +176,7 @@ $(cat graphify-out/.graphify_python) -c " import sys, json from graphify.extract import collect_files, extract from pathlib import Path +import json code_files = [] detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) @@ -183,13 +184,13 @@ for f in detect.get('files', {}).get('code', []): code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) if code_files: - result = extract(code_files, cache_root=Path(sys.argv[1])) + result = extract(code_files, cache_root=Path('INPUT_PATH')) Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') else: Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") print('No code files - skipping AST extraction') -" "INPUT_PATH" +" ``` #### Part B - Semantic extraction (parallel subagents) @@ -204,7 +205,7 @@ Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],' " ``` -**MANDATORY: You MUST use the subagent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the subagent tool you are doing this wrong.** +**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** Before dispatching subagents, print a timing estimate: - Load `total_words` and file counts from `graphify-out/.graphify_detect.json` @@ -220,7 +221,7 @@ SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from graphify.cache import check_semantic_cache from pathlib import Path @@ -230,7 +231,7 @@ detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encodin # every source file (#1392). Video is transcribed to a document in Step 2.5 first. all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root=sys.argv[1], prompt_file=sys.argv[2]) +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') # Always (re)write the cache file: write hits, else DELETE any leftover from a prior # run so Part C never merges a stale .graphify_cached.json (#1392). @@ -240,7 +241,7 @@ else: Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" "INPUT_PATH" "SPEC_PATH" +" ``` Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. @@ -283,7 +284,7 @@ Wait for all subagents. For each result: If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. -Merge all chunk files into `.graphify_semantic_new.json`. **After each subagent call completes, read the real token counts from the subagent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: +Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: ```bash $(cat graphify-out/.graphify_python) -c " import json, glob @@ -310,15 +311,15 @@ print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens' Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root=sys.argv[1], allowed_source_files=uncached, prompt_file=sys.argv[2]) +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') print(f'Cached {saved} files') -" "INPUT_PATH" "SPEC_PATH" +" ``` Merge cached + new results into `graphify-out/.graphify_semantic.json`: @@ -407,7 +408,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -432,7 +433,7 @@ if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') print('If this shrink is intentional (you deleted files), re-run a full build with --force.') raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, sys.argv[1], suggested_questions=questions) +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") analysis = { 'communities': {str(k): v for k, v in communities.items()}, @@ -443,7 +444,7 @@ analysis = { } Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" "INPUT_PATH" +" ``` If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. @@ -456,12 +457,12 @@ A non-destructive diagnostic on the extraction, before labeling. It surfaces edg ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from pathlib import Path from graphify.diagnostics import diagnose_extraction, format_diagnostic_report extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root=sys.argv[1]) +summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( ('dangling_endpoint_edges', 'dangling-endpoint edges'), @@ -471,7 +472,7 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0)] print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') -" "INPUT_PATH" +" ``` Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules). @@ -497,7 +498,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} @@ -508,7 +509,9 @@ labels = LABELS_DICT # Regenerate questions with real community labels (labels affect question phrasing) questions = suggest_questions(G, communities, labels) -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, sys.argv[1], suggested_questions=questions) +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") +Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") # Re-export so graph.json nodes carry the curated community_name (#2490). # Same extraction as Step 4, so the #479 shrink-guard passes on node count; # if it still refuses, surface the guard message - do not force past it. @@ -516,11 +519,8 @@ wrote = to_json(G, communities, 'graphify-out/graph.json', community_labels=labe if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') print('If this shrink is intentional (you deleted files), re-run a full build with --force.') - raise SystemExit(1) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") print('Report updated with community labels') -" "INPUT_PATH" +" ``` Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). @@ -535,14 +535,15 @@ If `--obsidian` was given: - If `--obsidian-dir ` was also given, pass it via `--dir`. Otherwise defaults to `graphify-out/obsidian`. ```bash -$(cat graphify-out/.graphify_python) -m graphify export obsidian -# Add `--dir ~/vaults/my-project` to use a custom directory. +graphify export obsidian +# or with custom dir: graphify export obsidian --dir ~/vaults/my-project ``` Generate the HTML graph (always, unless `--no-viz`): ```bash -$(cat graphify-out/.graphify_python) -m graphify export html # auto-aggregates to community view if graph > 5000 nodes +graphify export html # auto-aggregates to community view if graph > 5000 nodes +# or: graphify export html --no-viz ``` ### Steps 6b-8 - Wiki, Neo4j, FalkorDB, SVG, GraphML, MCP, benchmark (only on their flags) @@ -555,7 +556,7 @@ These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, ` ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from pathlib import Path from datetime import datetime, timezone from graphify.detect import save_manifest @@ -578,7 +579,7 @@ extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encod # types are gated on output. from graphify.cli import _stamped_manifest_files _corpus = detect.get('all_files') or detect['files'] -_manifest_files = _stamped_manifest_files(_corpus, extract, Path(sys.argv[1])) +_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) # Files dispatched this run (the changed subset) but NOT stamped above still carry # a stale semantic_hash from a prior run; clear it so detect_incremental re-queues # them instead of reading them as unchanged (#1948). @@ -590,7 +591,7 @@ _cleared = _dispatched - _stamped # files newly excluded since last run are dropped rather than masquerading as # deletions; untouched files' prior rows are still preserved (#1908). _scan = {f for fl in _corpus.values() for f in fl} -save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) # Update cumulative cost tracker input_tok = extract.get('input_tokens', 0) @@ -614,10 +615,10 @@ cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\" print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" "INPUT_PATH" +" rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null -rm -f graphify-out/needs_update 2>/dev/null || true +rm -f graphify-out/.needs_update 2>/dev/null || true ``` Replace INPUT_PATH with the actual path (same value used in Steps 4-5) so the manifest is relativized to the scan root. @@ -682,7 +683,7 @@ Both are non-default subcommands. `--update` re-extracts only new or changed fil When `graphify-out/graph.json` already exists and the user asks a question about the corpus, answer from the graph rather than rebuilding it: ```bash -$(cat graphify-out/.graphify_python) -m graphify query "" +graphify query "" ``` Before traversal, expand the question against the graph's own vocabulary so a wording mismatch does not collapse the answer to noise. If the `graphify query` CLI is unavailable, fall back to an inline NetworkX traversal of `graphify-out/graph.json`. Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. For that vocab-expansion step, the BFS/DFS traversal modes, the `--budget` cap, the NetworkX fallback, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`. diff --git a/tools/skillgen/expected/graphify__skill-vscode.md b/tools/skillgen/expected/graphify__skill-vscode.md index a251e12c02..20c7c0835c 100644 --- a/tools/skillgen/expected/graphify__skill-vscode.md +++ b/tools/skillgen/expected/graphify__skill-vscode.md @@ -97,7 +97,7 @@ fi mkdir -p graphify-out "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" # Save scan root so `graphify update` (no args) knows where to look next time -echo "$(cd "INPUT_PATH" && pwd)" > graphify-out/.graphify_root +echo "$(cd INPUT_PATH && pwd)" > graphify-out/.graphify_root ``` If the import succeeds, print nothing and move straight to Step 2. @@ -108,15 +108,15 @@ If the import succeeds, print nothing and move straight to Step 2. ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from graphify.detect import detect from pathlib import Path -result = detect(Path(sys.argv[1])) +result = detect(Path('INPUT_PATH')) # Write the sidecar from Python, not a shell redirect, so the same block renders # on PowerShell hosts without console-encoding drift (#2528). Path('graphify-out/.graphify_detect.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") print(f'Detected {result[\"total_files\"]} files') -" "INPUT_PATH" +" ``` Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: @@ -176,6 +176,7 @@ $(cat graphify-out/.graphify_python) -c " import sys, json from graphify.extract import collect_files, extract from pathlib import Path +import json code_files = [] detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) @@ -183,13 +184,13 @@ for f in detect.get('files', {}).get('code', []): code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) if code_files: - result = extract(code_files, cache_root=Path(sys.argv[1])) + result = extract(code_files, cache_root=Path('INPUT_PATH')) Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') else: Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") print('No code files - skipping AST extraction') -" "INPUT_PATH" +" ``` #### Part B - Semantic extraction (parallel subagents) @@ -204,7 +205,7 @@ Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],' " ``` -**MANDATORY: You MUST use the subagent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the subagent tool you are doing this wrong.** +**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** Before dispatching subagents, print a timing estimate: - Load `total_words` and file counts from `graphify-out/.graphify_detect.json` @@ -220,7 +221,7 @@ SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from graphify.cache import check_semantic_cache from pathlib import Path @@ -230,7 +231,7 @@ detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encodin # every source file (#1392). Video is transcribed to a document in Step 2.5 first. all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root=sys.argv[1], prompt_file=sys.argv[2]) +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') # Always (re)write the cache file: write hits, else DELETE any leftover from a prior # run so Part C never merges a stale .graphify_cached.json (#1392). @@ -240,7 +241,7 @@ else: Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" "INPUT_PATH" "SPEC_PATH" +" ``` Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. @@ -281,7 +282,7 @@ Wait for all subagents. For each result: If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. -Merge all chunk files into `.graphify_semantic_new.json`. **After each subagent call completes, read the real token counts from the subagent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: +Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: ```bash $(cat graphify-out/.graphify_python) -c " import json, glob @@ -308,15 +309,15 @@ print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens' Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root=sys.argv[1], allowed_source_files=uncached, prompt_file=sys.argv[2]) +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') print(f'Cached {saved} files') -" "INPUT_PATH" "SPEC_PATH" +" ``` Merge cached + new results into `graphify-out/.graphify_semantic.json`: @@ -405,7 +406,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -430,7 +431,7 @@ if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') print('If this shrink is intentional (you deleted files), re-run a full build with --force.') raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, sys.argv[1], suggested_questions=questions) +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") analysis = { 'communities': {str(k): v for k, v in communities.items()}, @@ -441,7 +442,7 @@ analysis = { } Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" "INPUT_PATH" +" ``` If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. @@ -454,12 +455,12 @@ A non-destructive diagnostic on the extraction, before labeling. It surfaces edg ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from pathlib import Path from graphify.diagnostics import diagnose_extraction, format_diagnostic_report extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root=sys.argv[1]) +summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( ('dangling_endpoint_edges', 'dangling-endpoint edges'), @@ -469,7 +470,7 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0)] print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') -" "INPUT_PATH" +" ``` Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules). @@ -495,7 +496,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} @@ -506,7 +507,9 @@ labels = LABELS_DICT # Regenerate questions with real community labels (labels affect question phrasing) questions = suggest_questions(G, communities, labels) -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, sys.argv[1], suggested_questions=questions) +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") +Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") # Re-export so graph.json nodes carry the curated community_name (#2490). # Same extraction as Step 4, so the #479 shrink-guard passes on node count; # if it still refuses, surface the guard message - do not force past it. @@ -514,11 +517,8 @@ wrote = to_json(G, communities, 'graphify-out/graph.json', community_labels=labe if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') print('If this shrink is intentional (you deleted files), re-run a full build with --force.') - raise SystemExit(1) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") print('Report updated with community labels') -" "INPUT_PATH" +" ``` Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). @@ -533,14 +533,15 @@ If `--obsidian` was given: - If `--obsidian-dir ` was also given, pass it via `--dir`. Otherwise defaults to `graphify-out/obsidian`. ```bash -$(cat graphify-out/.graphify_python) -m graphify export obsidian -# Add `--dir ~/vaults/my-project` to use a custom directory. +graphify export obsidian +# or with custom dir: graphify export obsidian --dir ~/vaults/my-project ``` Generate the HTML graph (always, unless `--no-viz`): ```bash -$(cat graphify-out/.graphify_python) -m graphify export html # auto-aggregates to community view if graph > 5000 nodes +graphify export html # auto-aggregates to community view if graph > 5000 nodes +# or: graphify export html --no-viz ``` ### Steps 6b-8 - Wiki, Neo4j, FalkorDB, SVG, GraphML, MCP, benchmark (only on their flags) @@ -553,7 +554,7 @@ These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, ` ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from pathlib import Path from datetime import datetime, timezone from graphify.detect import save_manifest @@ -576,7 +577,7 @@ extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encod # types are gated on output. from graphify.cli import _stamped_manifest_files _corpus = detect.get('all_files') or detect['files'] -_manifest_files = _stamped_manifest_files(_corpus, extract, Path(sys.argv[1])) +_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) # Files dispatched this run (the changed subset) but NOT stamped above still carry # a stale semantic_hash from a prior run; clear it so detect_incremental re-queues # them instead of reading them as unchanged (#1948). @@ -588,7 +589,7 @@ _cleared = _dispatched - _stamped # files newly excluded since last run are dropped rather than masquerading as # deletions; untouched files' prior rows are still preserved (#1908). _scan = {f for fl in _corpus.values() for f in fl} -save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) # Update cumulative cost tracker input_tok = extract.get('input_tokens', 0) @@ -612,10 +613,10 @@ cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\" print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" "INPUT_PATH" +" rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null -rm -f graphify-out/needs_update 2>/dev/null || true +rm -f graphify-out/.needs_update 2>/dev/null || true ``` Replace INPUT_PATH with the actual path (same value used in Steps 4-5) so the manifest is relativized to the scan root. @@ -680,7 +681,7 @@ Both are non-default subcommands. `--update` re-extracts only new or changed fil When `graphify-out/graph.json` already exists and the user asks a question about the corpus, answer from the graph rather than rebuilding it: ```bash -$(cat graphify-out/.graphify_python) -m graphify query "" +graphify query "" ``` Before traversal, expand the question against the graph's own vocabulary so a wording mismatch does not collapse the answer to noise. If the `graphify query` CLI is unavailable, fall back to an inline NetworkX traversal of `graphify-out/graph.json`. Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. For that vocab-expansion step, the BFS/DFS traversal modes, the `--budget` cap, the NetworkX fallback, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`. diff --git a/tools/skillgen/expected/graphify__skill-windows.md b/tools/skillgen/expected/graphify__skill-windows.md index fbbf2277ab..b09ecca3c4 100644 --- a/tools/skillgen/expected/graphify__skill-windows.md +++ b/tools/skillgen/expected/graphify__skill-windows.md @@ -124,7 +124,7 @@ if (-not $GRAPHIFY_PYTHON) { $Utf8NoBom = New-Object System.Text.UTF8Encoding $false [System.IO.File]::WriteAllText((Join-Path $PWD 'graphify-out\.graphify_python'), [string]$GRAPHIFY_PYTHON, $Utf8NoBom) # Save scan root so `graphify update` (no args) knows where to look next time -[System.IO.File]::WriteAllText((Join-Path $PWD 'graphify-out\.graphify_root'), (Resolve-Path "INPUT_PATH").Path, $Utf8NoBom) +[System.IO.File]::WriteAllText((Join-Path $PWD 'graphify-out\.graphify_root'), (Resolve-Path INPUT_PATH).Path, $Utf8NoBom) ``` If the import succeeds, print nothing and move straight to Step 2. @@ -135,15 +135,15 @@ If the import succeeds, print nothing and move straight to Step 2. ```powershell @' -import sys, json +import json from graphify.detect import detect from pathlib import Path -result = detect(Path(sys.argv[1])) +result = detect(Path('INPUT_PATH')) # Write the sidecar from Python, not a shell redirect, so the same block renders # on PowerShell hosts without console-encoding drift (#2528). Path('graphify-out/.graphify_detect.json').write_text(json.dumps(result, ensure_ascii=False), encoding="utf-8") print(f'Detected {result["total_files"]} files') -'@ | & (Get-Content graphify-out\.graphify_python) - "INPUT_PATH" +'@ | & (Get-Content graphify-out\.graphify_python) - ``` Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: @@ -203,6 +203,7 @@ For any code files detected, run AST extraction in parallel with Part B subagent import sys, json from graphify.extract import collect_files, extract from pathlib import Path +import json code_files = [] detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding="utf-8")) @@ -210,13 +211,13 @@ for f in detect.get('files', {}).get('code', []): code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) if code_files: - result = extract(code_files, cache_root=Path(sys.argv[1])) + result = extract(code_files, cache_root=Path('INPUT_PATH')) Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding="utf-8") print(f'AST: {len(result["nodes"])} nodes, {len(result["edges"])} edges') else: Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding="utf-8") print('No code files - skipping AST extraction') -'@ | & (Get-Content graphify-out\.graphify_python) - "INPUT_PATH" +'@ | & (Get-Content graphify-out\.graphify_python) - ``` #### Part B - Semantic extraction (parallel subagents) @@ -231,7 +232,7 @@ Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],' '@ | & (Get-Content graphify-out\.graphify_python) - ``` -**MANDATORY: You MUST use the subagent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the subagent tool you are doing this wrong.** +**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** Before dispatching subagents, print a timing estimate: - Load `total_words` and file counts from `graphify-out/.graphify_detect.json` @@ -247,7 +248,7 @@ SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` ```powershell @' -import sys, json +import json from graphify.cache import check_semantic_cache from pathlib import Path @@ -257,7 +258,7 @@ detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encodin # every source file (#1392). Video is transcribed to a document in Step 2.5 first. all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root=sys.argv[1], prompt_file=sys.argv[2]) +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') # Always (re)write the cache file: write hits, else DELETE any leftover from a prior # run so Part C never merges a stale .graphify_cached.json (#1392). @@ -267,7 +268,7 @@ else: Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding="utf-8") print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -'@ | & (Get-Content graphify-out\.graphify_python) - "INPUT_PATH" "SPEC_PATH" +'@ | & (Get-Content graphify-out\.graphify_python) - ``` Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. @@ -312,7 +313,7 @@ Wait for all subagents. For each result: If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. -Merge all chunk files into `.graphify_semantic_new.json`. **After each subagent call completes, read the real token counts from the subagent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: +Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: ```powershell @' import json, glob @@ -339,15 +340,15 @@ print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens' Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): ```powershell @' -import sys, json +import json from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding="utf-8")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding="utf-8").splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root=sys.argv[1], allowed_source_files=uncached, prompt_file=sys.argv[2]) +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') print(f'Cached {saved} files') -'@ | & (Get-Content graphify-out\.graphify_python) - "INPUT_PATH" "SPEC_PATH" +'@ | & (Get-Content graphify-out\.graphify_python) - ``` Merge cached + new results into `graphify-out/.graphify_semantic.json`: @@ -436,7 +437,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -461,7 +462,7 @@ if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') print('If this shrink is intentional (you deleted files), re-run a full build with --force.') raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, sys.argv[1], suggested_questions=questions) +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding="utf-8") analysis = { 'communities': {str(k): v for k, v in communities.items()}, @@ -472,7 +473,7 @@ analysis = { } Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding="utf-8") print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -'@ | & (Get-Content graphify-out\.graphify_python) - "INPUT_PATH" +'@ | & (Get-Content graphify-out\.graphify_python) - ``` If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. @@ -485,12 +486,12 @@ A non-destructive diagnostic on the extraction, before labeling. It surfaces edg ```powershell @' -import sys, json +import json from pathlib import Path from graphify.diagnostics import diagnose_extraction, format_diagnostic_report extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding="utf-8")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root=sys.argv[1]) +summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( ('dangling_endpoint_edges', 'dangling-endpoint edges'), @@ -500,7 +501,7 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0)] print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') -'@ | & (Get-Content graphify-out\.graphify_python) - "INPUT_PATH" +'@ | & (Get-Content graphify-out\.graphify_python) - ``` Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules). @@ -526,7 +527,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding="utf-8")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} @@ -537,7 +538,9 @@ labels = LABELS_DICT # Regenerate questions with real community labels (labels affect question phrasing) questions = suggest_questions(G, communities, labels) -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, sys.argv[1], suggested_questions=questions) +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding="utf-8") +Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding="utf-8") # Re-export so graph.json nodes carry the curated community_name (#2490). # Same extraction as Step 4, so the #479 shrink-guard passes on node count; # if it still refuses, surface the guard message - do not force past it. @@ -545,11 +548,8 @@ wrote = to_json(G, communities, 'graphify-out/graph.json', community_labels=labe if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') print('If this shrink is intentional (you deleted files), re-run a full build with --force.') - raise SystemExit(1) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding="utf-8") -Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding="utf-8") print('Report updated with community labels') -'@ | & (Get-Content graphify-out\.graphify_python) - "INPUT_PATH" +'@ | & (Get-Content graphify-out\.graphify_python) - ``` Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). @@ -564,14 +564,15 @@ If `--obsidian` was given: - If `--obsidian-dir ` was also given, pass it via `--dir`. Otherwise defaults to `graphify-out/obsidian`. ```powershell -& (Get-Content graphify-out\.graphify_python) -m graphify export obsidian -# Add `--dir ~/vaults/my-project` to use a custom directory. +graphify export obsidian +# or with custom dir: graphify export obsidian --dir ~/vaults/my-project ``` Generate the HTML graph (always, unless `--no-viz`): ```powershell -& (Get-Content graphify-out\.graphify_python) -m graphify export html # auto-aggregates to community view if graph > 5000 nodes +graphify export html # auto-aggregates to community view if graph > 5000 nodes +# or: graphify export html --no-viz ``` ### Steps 6b-8 - Wiki, Neo4j, FalkorDB, SVG, GraphML, MCP, benchmark (only on their flags) @@ -584,7 +585,7 @@ These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, ` ```powershell @' -import sys, json +import json from pathlib import Path from datetime import datetime, timezone from graphify.detect import save_manifest @@ -607,7 +608,7 @@ extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encod # types are gated on output. from graphify.cli import _stamped_manifest_files _corpus = detect.get('all_files') or detect['files'] -_manifest_files = _stamped_manifest_files(_corpus, extract, Path(sys.argv[1])) +_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) # Files dispatched this run (the changed subset) but NOT stamped above still carry # a stale semantic_hash from a prior run; clear it so detect_incremental re-queues # them instead of reading them as unchanged (#1948). @@ -619,7 +620,7 @@ _cleared = _dispatched - _stamped # files newly excluded since last run are dropped rather than masquerading as # deletions; untouched files' prior rows are still preserved (#1908). _scan = {f for fl in _corpus.values() for f in fl} -save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) # Update cumulative cost tracker input_tok = extract.get('input_tokens', 0) @@ -643,10 +644,10 @@ cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding="u print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') print(f'All time: {cost["total_input_tokens"]:,} input, {cost["total_output_tokens"]:,} output ({len(cost["runs"])} runs)') -'@ | & (Get-Content graphify-out\.graphify_python) - "INPUT_PATH" +'@ | & (Get-Content graphify-out\.graphify_python) - Remove-Item -Force -ErrorAction SilentlyContinue graphify-out\.graphify_detect.json, graphify-out\.graphify_extract.json, graphify-out\.graphify_ast.json, graphify-out\.graphify_semantic.json, graphify-out\.graphify_analysis.json Get-ChildItem graphify-out -Filter '.graphify_chunk_*.json' -File -ErrorAction SilentlyContinue | Remove-Item -Force -Remove-Item -Force -ErrorAction SilentlyContinue graphify-out\needs_update +Remove-Item -Force -ErrorAction SilentlyContinue graphify-out\.needs_update ``` Replace INPUT_PATH with the actual path (same value used in Steps 4-5) so the manifest is relativized to the scan root. @@ -713,7 +714,7 @@ Both are non-default subcommands. `--update` re-extracts only new or changed fil When `graphify-out/graph.json` already exists and the user asks a question about the corpus, answer from the graph rather than rebuilding it: ```powershell -& (Get-Content graphify-out\.graphify_python) -m graphify query "" +graphify query "" ``` Before traversal, expand the question against the graph's own vocabulary so a wording mismatch does not collapse the answer to noise. If the `graphify query` CLI is unavailable, fall back to an inline NetworkX traversal of `graphify-out/graph.json`. Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. For that vocab-expansion step, the BFS/DFS traversal modes, the `--budget` cap, the NetworkX fallback, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`. diff --git a/tools/skillgen/expected/graphify__skill.md b/tools/skillgen/expected/graphify__skill.md index 55b77d6828..abd2811d23 100644 --- a/tools/skillgen/expected/graphify__skill.md +++ b/tools/skillgen/expected/graphify__skill.md @@ -97,7 +97,7 @@ fi mkdir -p graphify-out "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" # Save scan root so `graphify update` (no args) knows where to look next time -echo "$(cd "INPUT_PATH" && pwd)" > graphify-out/.graphify_root +echo "$(cd INPUT_PATH && pwd)" > graphify-out/.graphify_root ``` If the import succeeds, print nothing and move straight to Step 2. @@ -108,15 +108,15 @@ If the import succeeds, print nothing and move straight to Step 2. ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from graphify.detect import detect from pathlib import Path -result = detect(Path(sys.argv[1])) +result = detect(Path('INPUT_PATH')) # Write the sidecar from Python, not a shell redirect, so the same block renders # on PowerShell hosts without console-encoding drift (#2528). Path('graphify-out/.graphify_detect.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") print(f'Detected {result[\"total_files\"]} files') -" "INPUT_PATH" +" ``` Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: @@ -176,6 +176,7 @@ $(cat graphify-out/.graphify_python) -c " import sys, json from graphify.extract import collect_files, extract from pathlib import Path +import json code_files = [] detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) @@ -183,13 +184,13 @@ for f in detect.get('files', {}).get('code', []): code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) if code_files: - result = extract(code_files, cache_root=Path(sys.argv[1])) + result = extract(code_files, cache_root=Path('INPUT_PATH')) Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') else: Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") print('No code files - skipping AST extraction') -" "INPUT_PATH" +" ``` #### Part B - Semantic extraction (parallel subagents) @@ -204,7 +205,7 @@ Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],' " ``` -**MANDATORY: You MUST use the subagent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the subagent tool you are doing this wrong.** +**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** Before dispatching subagents, print a timing estimate: - Load `total_words` and file counts from `graphify-out/.graphify_detect.json` @@ -220,7 +221,7 @@ SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from graphify.cache import check_semantic_cache from pathlib import Path @@ -230,7 +231,7 @@ detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encodin # every source file (#1392). Video is transcribed to a document in Step 2.5 first. all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root=sys.argv[1], prompt_file=sys.argv[2]) +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') # Always (re)write the cache file: write hits, else DELETE any leftover from a prior # run so Part C never merges a stale .graphify_cached.json (#1392). @@ -240,7 +241,7 @@ else: Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" "INPUT_PATH" "SPEC_PATH" +" ``` Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. @@ -285,7 +286,7 @@ Wait for all subagents. For each result: If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. -Merge all chunk files into `.graphify_semantic_new.json`. **After each subagent call completes, read the real token counts from the subagent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: +Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: ```bash $(cat graphify-out/.graphify_python) -c " import json, glob @@ -312,15 +313,15 @@ print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens' Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root=sys.argv[1], allowed_source_files=uncached, prompt_file=sys.argv[2]) +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') print(f'Cached {saved} files') -" "INPUT_PATH" "SPEC_PATH" +" ``` Merge cached + new results into `graphify-out/.graphify_semantic.json`: @@ -409,7 +410,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -434,7 +435,7 @@ if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') print('If this shrink is intentional (you deleted files), re-run a full build with --force.') raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, sys.argv[1], suggested_questions=questions) +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") analysis = { 'communities': {str(k): v for k, v in communities.items()}, @@ -445,7 +446,7 @@ analysis = { } Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" "INPUT_PATH" +" ``` If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. @@ -458,12 +459,12 @@ A non-destructive diagnostic on the extraction, before labeling. It surfaces edg ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from pathlib import Path from graphify.diagnostics import diagnose_extraction, format_diagnostic_report extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root=sys.argv[1]) +summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( ('dangling_endpoint_edges', 'dangling-endpoint edges'), @@ -473,7 +474,7 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0)] print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') -" "INPUT_PATH" +" ``` Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules). @@ -499,7 +500,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} @@ -510,7 +511,9 @@ labels = LABELS_DICT # Regenerate questions with real community labels (labels affect question phrasing) questions = suggest_questions(G, communities, labels) -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, sys.argv[1], suggested_questions=questions) +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") +Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") # Re-export so graph.json nodes carry the curated community_name (#2490). # Same extraction as Step 4, so the #479 shrink-guard passes on node count; # if it still refuses, surface the guard message - do not force past it. @@ -518,11 +521,8 @@ wrote = to_json(G, communities, 'graphify-out/graph.json', community_labels=labe if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') print('If this shrink is intentional (you deleted files), re-run a full build with --force.') - raise SystemExit(1) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") print('Report updated with community labels') -" "INPUT_PATH" +" ``` Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). @@ -537,14 +537,15 @@ If `--obsidian` was given: - If `--obsidian-dir ` was also given, pass it via `--dir`. Otherwise defaults to `graphify-out/obsidian`. ```bash -$(cat graphify-out/.graphify_python) -m graphify export obsidian -# Add `--dir ~/vaults/my-project` to use a custom directory. +graphify export obsidian +# or with custom dir: graphify export obsidian --dir ~/vaults/my-project ``` Generate the HTML graph (always, unless `--no-viz`): ```bash -$(cat graphify-out/.graphify_python) -m graphify export html # auto-aggregates to community view if graph > 5000 nodes +graphify export html # auto-aggregates to community view if graph > 5000 nodes +# or: graphify export html --no-viz ``` ### Steps 6b-8 - Wiki, Neo4j, FalkorDB, SVG, GraphML, MCP, benchmark (only on their flags) @@ -557,7 +558,7 @@ These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, ` ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from pathlib import Path from datetime import datetime, timezone from graphify.detect import save_manifest @@ -580,7 +581,7 @@ extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encod # types are gated on output. from graphify.cli import _stamped_manifest_files _corpus = detect.get('all_files') or detect['files'] -_manifest_files = _stamped_manifest_files(_corpus, extract, Path(sys.argv[1])) +_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) # Files dispatched this run (the changed subset) but NOT stamped above still carry # a stale semantic_hash from a prior run; clear it so detect_incremental re-queues # them instead of reading them as unchanged (#1948). @@ -592,7 +593,7 @@ _cleared = _dispatched - _stamped # files newly excluded since last run are dropped rather than masquerading as # deletions; untouched files' prior rows are still preserved (#1908). _scan = {f for fl in _corpus.values() for f in fl} -save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) # Update cumulative cost tracker input_tok = extract.get('input_tokens', 0) @@ -616,10 +617,10 @@ cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\" print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" "INPUT_PATH" +" rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null -rm -f graphify-out/needs_update 2>/dev/null || true +rm -f graphify-out/.needs_update 2>/dev/null || true ``` Replace INPUT_PATH with the actual path (same value used in Steps 4-5) so the manifest is relativized to the scan root. @@ -684,7 +685,7 @@ Both are non-default subcommands. `--update` re-extracts only new or changed fil When `graphify-out/graph.json` already exists and the user asks a question about the corpus, answer from the graph rather than rebuilding it: ```bash -$(cat graphify-out/.graphify_python) -m graphify query "" +graphify query "" ``` Before traversal, expand the question against the graph's own vocabulary so a wording mismatch does not collapse the answer to noise. If the `graphify query` CLI is unavailable, fall back to an inline NetworkX traversal of `graphify-out/graph.json`. Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. For that vocab-expansion step, the BFS/DFS traversal modes, the `--budget` cap, the NetworkX fallback, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`. diff --git a/tools/skillgen/expected/graphify__skills__agents__references__add-watch.md b/tools/skillgen/expected/graphify__skills__agents__references__add-watch.md index fcbe0cd488..77844343e1 100644 --- a/tools/skillgen/expected/graphify__skills__agents__references__add-watch.md +++ b/tools/skillgen/expected/graphify__skills__agents__references__add-watch.md @@ -13,7 +13,7 @@ from graphify.ingest import ingest from pathlib import Path try: - out = ingest(sys.argv[1], Path('./raw'), author=sys.argv[2] or None, contributor=sys.argv[3] or None) + out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') print(f'Saved to {out}') except ValueError as e: print(f'error: {e}', file=sys.stderr) @@ -21,10 +21,10 @@ except ValueError as e: except RuntimeError as e: print(f'error: {e}', file=sys.stderr) sys.exit(1) -" "URL" "AUTHOR" "CONTRIBUTOR" +" ``` -Replace `URL` with the actual URL and pass the user's `AUTHOR` or `CONTRIBUTOR` when provided; use an empty string for either omitted value. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on the scan root recorded in `graphify-out/.graphify_root` to merge the new file into the existing graph. +Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. Supported URL types (auto-detected): - YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`) @@ -41,7 +41,7 @@ Supported URL types (auto-detected): Start a background watcher that monitors a folder and auto-updates the graph when files change. ```bash -$(cat graphify-out/.graphify_python) -m graphify.watch "INPUT_PATH" --debounce 3 +$(cat graphify-out/.graphify_python) -m graphify.watch INPUT_PATH --debounce 3 ``` Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: diff --git a/tools/skillgen/expected/graphify__skills__agents__references__exports.md b/tools/skillgen/expected/graphify__skills__agents__references__exports.md index 89ce996fb7..242ff868e0 100644 --- a/tools/skillgen/expected/graphify__skills__agents__references__exports.md +++ b/tools/skillgen/expected/graphify__skills__agents__references__exports.md @@ -20,27 +20,13 @@ graphify export wiki graphify export neo4j ``` -**If `--neo4j-push `** - push directly to a running Neo4j instance. Do not run these blocks through an agent tool: its shell is non-interactive and cannot read a hidden password. Show the block for the user's shell, ask them to run it in their own terminal, and wait for confirmation before continuing. Never ask them to paste the password into chat. - -POSIX shell: +**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: ```bash -NEO4J_PASSWORD="$("$(cat graphify-out/.graphify_python)" -c 'import getpass; print(getpass.getpass("Neo4j password: "))')" -export NEO4J_PASSWORD -graphify export neo4j --push bolt://localhost:7687 --user neo4j -unset NEO4J_PASSWORD -``` - -PowerShell: - -```powershell -$credential = Read-Host "Neo4j password" -AsSecureString -$env:NEO4J_PASSWORD = [System.Net.NetworkCredential]::new("", $credential).Password -graphify export neo4j --push bolt://localhost:7687 --user neo4j -Remove-Item Env:NEO4J_PASSWORD +graphify export neo4j --push bolt://localhost:7687 --user neo4j --password PASSWORD ``` -Default URI is `bolt://localhost:7687`, default user is `neo4j`. The command reads the password from `NEO4J_PASSWORD`. Uses MERGE - safe to re-run without creating duplicates. +Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. ### Step 7a - FalkorDB export (only if --falkordb or --falkordb-push flag) diff --git a/tools/skillgen/expected/graphify__skills__agents__references__extraction-spec.md b/tools/skillgen/expected/graphify__skills__agents__references__extraction-spec.md index a5a12a575e..388df7674f 100644 --- a/tools/skillgen/expected/graphify__skills__agents__references__extraction-spec.md +++ b/tools/skillgen/expected/graphify__skills__agents__references__extraction-spec.md @@ -61,7 +61,7 @@ confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a d Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is the **full repo-relative path with the extension dropped**, every path segment kept and joined with `_` (each segment lowercased with non-alphanumeric chars replaced by `_`), and entity is the symbol name similarly normalized. Use every directory level, not just the immediate parent — this keeps same-named files in different directories distinct. Examples: `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `lib_utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`; `docs/v1/api/README.md` + `getUser` → `docs_v1_api_readme_getuser`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or only the immediate parent (e.g., `auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project built under the old immediate-parent format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. Generate the extraction JSON matching this schema exactly: -{"nodes":[{"id":"src_auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} +{"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. diff --git a/tools/skillgen/expected/graphify__skills__agents__references__query.md b/tools/skillgen/expected/graphify__skills__agents__references__query.md index 6dd627863c..56565eb782 100644 --- a/tools/skillgen/expected/graphify__skills__agents__references__query.md +++ b/tools/skillgen/expected/graphify__skills__agents__references__query.md @@ -79,12 +79,12 @@ If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal ```bash $(cat graphify-out/.graphify_python) -c " import sys, json -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph import networkx as nx from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' @@ -197,11 +197,11 @@ If the CLI is unavailable, run it inline: $(cat graphify-out/.graphify_python) -c " import json, sys import networkx as nx -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') a_term = 'NODE_A' b_term = 'NODE_B' @@ -265,11 +265,11 @@ If the CLI is unavailable, run it inline: $(cat graphify-out/.graphify_python) -c " import json, sys import networkx as nx -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') term = 'NODE_NAME' term_lower = term.lower() diff --git a/tools/skillgen/expected/graphify__skills__agents__references__transcribe.md b/tools/skillgen/expected/graphify__skills__agents__references__transcribe.md index bbe8b93e45..b967f83799 100644 --- a/tools/skillgen/expected/graphify__skills__agents__references__transcribe.md +++ b/tools/skillgen/expected/graphify__skills__agents__references__transcribe.md @@ -8,11 +8,13 @@ Skip this step entirely if `detect` returned zero `video` files. Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. -**Strategy:** If `graphify-out/.graphify_analysis.json` exists from a previous run, read its top god-node labels and write a one-sentence domain hint from them. Otherwise use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` Pass the chosen prompt to Whisper as its initial prompt. No separate API call needed. +**Strategy:** Read the god nodes from `graphify-out/.graphify_detect.json` (or the analysis file if it exists from a previous run). You are already a language model — write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. + +**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` **Step 1 - Write the Whisper prompt yourself.** -Read the top god-node labels from previous analysis when available, then compose a short domain hint sentence. With no previous analysis, use the generic fallback above. +Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: - Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` - Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` diff --git a/tools/skillgen/expected/graphify__skills__agents__references__update.md b/tools/skillgen/expected/graphify__skills__agents__references__update.md index 058ca8e6d2..3632fd4126 100644 --- a/tools/skillgen/expected/graphify__skills__agents__references__update.md +++ b/tools/skillgen/expected/graphify__skills__agents__references__update.md @@ -12,7 +12,7 @@ import sys, json from graphify.detect import detect_incremental, save_manifest from pathlib import Path -result = detect_incremental(Path(sys.argv[1])) +result = detect_incremental(Path('INPUT_PATH')) new_total = result.get('new_total', 0) print(json.dumps(result, indent=2, ensure_ascii=False)) Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") @@ -24,7 +24,7 @@ if deleted: print(f'{len(deleted)} deleted file(s) to prune.') if new_total > 0: print(f'{new_total} new/changed file(s) to re-extract.') -" "INPUT_PATH" +" ``` Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: @@ -84,7 +84,7 @@ Then: ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from pathlib import Path from graphify.build import build_merge from graphify.detect import save_manifest @@ -113,7 +113,7 @@ G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, - root=sys.argv[1], + root='INPUT_PATH', directed=IS_DIRECTED, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -150,7 +150,7 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) # is lost forever (#2015). Mirrors the library extract path # (cli._stamped_manifest_files + clear_semantic + scan_corpus). from graphify.cli import _stamped_manifest_files -_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path(sys.argv[1])) +_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) # Changed semantic files dispatched this run but NOT stamped had their chunk fail # or be omitted; clear any stale semantic_hash so they are re-queued (#1948). _sem_types = ('document', 'paper', 'image') @@ -160,9 +160,9 @@ _cleared = _dispatched - _stamped # scan_corpus = the RAW full corpus so in-root files newly excluded since last run # are dropped rather than masquerading as deletions; untouched rows preserved (#1908). _scan = {f for fl in incremental['files'].values() for f in fl} -save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) print('[graphify update] Manifest saved.') -" "INPUT_PATH" +" ``` Then run Steps 4–8 on the merged graph as normal. @@ -174,7 +174,7 @@ $(cat graphify-out/.graphify_python) -c " import json from graphify.analyze import graph_diff from graphify.build import build_from_json -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph import networkx as nx from pathlib import Path @@ -184,7 +184,7 @@ new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(e G_new = build_from_json(new_extract, directed=IS_DIRECTED) if old_data: - G_old = load_node_link_graph(old_data) + G_old = json_graph.node_link_graph(old_data, edges='links') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: diff --git a/tools/skillgen/expected/graphify__skills__amp__references__add-watch.md b/tools/skillgen/expected/graphify__skills__amp__references__add-watch.md index fcbe0cd488..77844343e1 100644 --- a/tools/skillgen/expected/graphify__skills__amp__references__add-watch.md +++ b/tools/skillgen/expected/graphify__skills__amp__references__add-watch.md @@ -13,7 +13,7 @@ from graphify.ingest import ingest from pathlib import Path try: - out = ingest(sys.argv[1], Path('./raw'), author=sys.argv[2] or None, contributor=sys.argv[3] or None) + out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') print(f'Saved to {out}') except ValueError as e: print(f'error: {e}', file=sys.stderr) @@ -21,10 +21,10 @@ except ValueError as e: except RuntimeError as e: print(f'error: {e}', file=sys.stderr) sys.exit(1) -" "URL" "AUTHOR" "CONTRIBUTOR" +" ``` -Replace `URL` with the actual URL and pass the user's `AUTHOR` or `CONTRIBUTOR` when provided; use an empty string for either omitted value. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on the scan root recorded in `graphify-out/.graphify_root` to merge the new file into the existing graph. +Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. Supported URL types (auto-detected): - YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`) @@ -41,7 +41,7 @@ Supported URL types (auto-detected): Start a background watcher that monitors a folder and auto-updates the graph when files change. ```bash -$(cat graphify-out/.graphify_python) -m graphify.watch "INPUT_PATH" --debounce 3 +$(cat graphify-out/.graphify_python) -m graphify.watch INPUT_PATH --debounce 3 ``` Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: diff --git a/tools/skillgen/expected/graphify__skills__amp__references__exports.md b/tools/skillgen/expected/graphify__skills__amp__references__exports.md index 89ce996fb7..242ff868e0 100644 --- a/tools/skillgen/expected/graphify__skills__amp__references__exports.md +++ b/tools/skillgen/expected/graphify__skills__amp__references__exports.md @@ -20,27 +20,13 @@ graphify export wiki graphify export neo4j ``` -**If `--neo4j-push `** - push directly to a running Neo4j instance. Do not run these blocks through an agent tool: its shell is non-interactive and cannot read a hidden password. Show the block for the user's shell, ask them to run it in their own terminal, and wait for confirmation before continuing. Never ask them to paste the password into chat. - -POSIX shell: +**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: ```bash -NEO4J_PASSWORD="$("$(cat graphify-out/.graphify_python)" -c 'import getpass; print(getpass.getpass("Neo4j password: "))')" -export NEO4J_PASSWORD -graphify export neo4j --push bolt://localhost:7687 --user neo4j -unset NEO4J_PASSWORD -``` - -PowerShell: - -```powershell -$credential = Read-Host "Neo4j password" -AsSecureString -$env:NEO4J_PASSWORD = [System.Net.NetworkCredential]::new("", $credential).Password -graphify export neo4j --push bolt://localhost:7687 --user neo4j -Remove-Item Env:NEO4J_PASSWORD +graphify export neo4j --push bolt://localhost:7687 --user neo4j --password PASSWORD ``` -Default URI is `bolt://localhost:7687`, default user is `neo4j`. The command reads the password from `NEO4J_PASSWORD`. Uses MERGE - safe to re-run without creating duplicates. +Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. ### Step 7a - FalkorDB export (only if --falkordb or --falkordb-push flag) diff --git a/tools/skillgen/expected/graphify__skills__amp__references__extraction-spec.md b/tools/skillgen/expected/graphify__skills__amp__references__extraction-spec.md index a5a12a575e..388df7674f 100644 --- a/tools/skillgen/expected/graphify__skills__amp__references__extraction-spec.md +++ b/tools/skillgen/expected/graphify__skills__amp__references__extraction-spec.md @@ -61,7 +61,7 @@ confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a d Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is the **full repo-relative path with the extension dropped**, every path segment kept and joined with `_` (each segment lowercased with non-alphanumeric chars replaced by `_`), and entity is the symbol name similarly normalized. Use every directory level, not just the immediate parent — this keeps same-named files in different directories distinct. Examples: `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `lib_utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`; `docs/v1/api/README.md` + `getUser` → `docs_v1_api_readme_getuser`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or only the immediate parent (e.g., `auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project built under the old immediate-parent format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. Generate the extraction JSON matching this schema exactly: -{"nodes":[{"id":"src_auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} +{"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. diff --git a/tools/skillgen/expected/graphify__skills__amp__references__query.md b/tools/skillgen/expected/graphify__skills__amp__references__query.md index 6dd627863c..56565eb782 100644 --- a/tools/skillgen/expected/graphify__skills__amp__references__query.md +++ b/tools/skillgen/expected/graphify__skills__amp__references__query.md @@ -79,12 +79,12 @@ If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal ```bash $(cat graphify-out/.graphify_python) -c " import sys, json -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph import networkx as nx from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' @@ -197,11 +197,11 @@ If the CLI is unavailable, run it inline: $(cat graphify-out/.graphify_python) -c " import json, sys import networkx as nx -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') a_term = 'NODE_A' b_term = 'NODE_B' @@ -265,11 +265,11 @@ If the CLI is unavailable, run it inline: $(cat graphify-out/.graphify_python) -c " import json, sys import networkx as nx -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') term = 'NODE_NAME' term_lower = term.lower() diff --git a/tools/skillgen/expected/graphify__skills__amp__references__transcribe.md b/tools/skillgen/expected/graphify__skills__amp__references__transcribe.md index bbe8b93e45..b967f83799 100644 --- a/tools/skillgen/expected/graphify__skills__amp__references__transcribe.md +++ b/tools/skillgen/expected/graphify__skills__amp__references__transcribe.md @@ -8,11 +8,13 @@ Skip this step entirely if `detect` returned zero `video` files. Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. -**Strategy:** If `graphify-out/.graphify_analysis.json` exists from a previous run, read its top god-node labels and write a one-sentence domain hint from them. Otherwise use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` Pass the chosen prompt to Whisper as its initial prompt. No separate API call needed. +**Strategy:** Read the god nodes from `graphify-out/.graphify_detect.json` (or the analysis file if it exists from a previous run). You are already a language model — write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. + +**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` **Step 1 - Write the Whisper prompt yourself.** -Read the top god-node labels from previous analysis when available, then compose a short domain hint sentence. With no previous analysis, use the generic fallback above. +Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: - Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` - Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` diff --git a/tools/skillgen/expected/graphify__skills__amp__references__update.md b/tools/skillgen/expected/graphify__skills__amp__references__update.md index 058ca8e6d2..3632fd4126 100644 --- a/tools/skillgen/expected/graphify__skills__amp__references__update.md +++ b/tools/skillgen/expected/graphify__skills__amp__references__update.md @@ -12,7 +12,7 @@ import sys, json from graphify.detect import detect_incremental, save_manifest from pathlib import Path -result = detect_incremental(Path(sys.argv[1])) +result = detect_incremental(Path('INPUT_PATH')) new_total = result.get('new_total', 0) print(json.dumps(result, indent=2, ensure_ascii=False)) Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") @@ -24,7 +24,7 @@ if deleted: print(f'{len(deleted)} deleted file(s) to prune.') if new_total > 0: print(f'{new_total} new/changed file(s) to re-extract.') -" "INPUT_PATH" +" ``` Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: @@ -84,7 +84,7 @@ Then: ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from pathlib import Path from graphify.build import build_merge from graphify.detect import save_manifest @@ -113,7 +113,7 @@ G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, - root=sys.argv[1], + root='INPUT_PATH', directed=IS_DIRECTED, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -150,7 +150,7 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) # is lost forever (#2015). Mirrors the library extract path # (cli._stamped_manifest_files + clear_semantic + scan_corpus). from graphify.cli import _stamped_manifest_files -_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path(sys.argv[1])) +_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) # Changed semantic files dispatched this run but NOT stamped had their chunk fail # or be omitted; clear any stale semantic_hash so they are re-queued (#1948). _sem_types = ('document', 'paper', 'image') @@ -160,9 +160,9 @@ _cleared = _dispatched - _stamped # scan_corpus = the RAW full corpus so in-root files newly excluded since last run # are dropped rather than masquerading as deletions; untouched rows preserved (#1908). _scan = {f for fl in incremental['files'].values() for f in fl} -save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) print('[graphify update] Manifest saved.') -" "INPUT_PATH" +" ``` Then run Steps 4–8 on the merged graph as normal. @@ -174,7 +174,7 @@ $(cat graphify-out/.graphify_python) -c " import json from graphify.analyze import graph_diff from graphify.build import build_from_json -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph import networkx as nx from pathlib import Path @@ -184,7 +184,7 @@ new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(e G_new = build_from_json(new_extract, directed=IS_DIRECTED) if old_data: - G_old = load_node_link_graph(old_data) + G_old = json_graph.node_link_graph(old_data, edges='links') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: diff --git a/tools/skillgen/expected/graphify__skills__claude__references__add-watch.md b/tools/skillgen/expected/graphify__skills__claude__references__add-watch.md index fcbe0cd488..77844343e1 100644 --- a/tools/skillgen/expected/graphify__skills__claude__references__add-watch.md +++ b/tools/skillgen/expected/graphify__skills__claude__references__add-watch.md @@ -13,7 +13,7 @@ from graphify.ingest import ingest from pathlib import Path try: - out = ingest(sys.argv[1], Path('./raw'), author=sys.argv[2] or None, contributor=sys.argv[3] or None) + out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') print(f'Saved to {out}') except ValueError as e: print(f'error: {e}', file=sys.stderr) @@ -21,10 +21,10 @@ except ValueError as e: except RuntimeError as e: print(f'error: {e}', file=sys.stderr) sys.exit(1) -" "URL" "AUTHOR" "CONTRIBUTOR" +" ``` -Replace `URL` with the actual URL and pass the user's `AUTHOR` or `CONTRIBUTOR` when provided; use an empty string for either omitted value. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on the scan root recorded in `graphify-out/.graphify_root` to merge the new file into the existing graph. +Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. Supported URL types (auto-detected): - YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`) @@ -41,7 +41,7 @@ Supported URL types (auto-detected): Start a background watcher that monitors a folder and auto-updates the graph when files change. ```bash -$(cat graphify-out/.graphify_python) -m graphify.watch "INPUT_PATH" --debounce 3 +$(cat graphify-out/.graphify_python) -m graphify.watch INPUT_PATH --debounce 3 ``` Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: diff --git a/tools/skillgen/expected/graphify__skills__claude__references__exports.md b/tools/skillgen/expected/graphify__skills__claude__references__exports.md index 89ce996fb7..242ff868e0 100644 --- a/tools/skillgen/expected/graphify__skills__claude__references__exports.md +++ b/tools/skillgen/expected/graphify__skills__claude__references__exports.md @@ -20,27 +20,13 @@ graphify export wiki graphify export neo4j ``` -**If `--neo4j-push `** - push directly to a running Neo4j instance. Do not run these blocks through an agent tool: its shell is non-interactive and cannot read a hidden password. Show the block for the user's shell, ask them to run it in their own terminal, and wait for confirmation before continuing. Never ask them to paste the password into chat. - -POSIX shell: +**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: ```bash -NEO4J_PASSWORD="$("$(cat graphify-out/.graphify_python)" -c 'import getpass; print(getpass.getpass("Neo4j password: "))')" -export NEO4J_PASSWORD -graphify export neo4j --push bolt://localhost:7687 --user neo4j -unset NEO4J_PASSWORD -``` - -PowerShell: - -```powershell -$credential = Read-Host "Neo4j password" -AsSecureString -$env:NEO4J_PASSWORD = [System.Net.NetworkCredential]::new("", $credential).Password -graphify export neo4j --push bolt://localhost:7687 --user neo4j -Remove-Item Env:NEO4J_PASSWORD +graphify export neo4j --push bolt://localhost:7687 --user neo4j --password PASSWORD ``` -Default URI is `bolt://localhost:7687`, default user is `neo4j`. The command reads the password from `NEO4J_PASSWORD`. Uses MERGE - safe to re-run without creating duplicates. +Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. ### Step 7a - FalkorDB export (only if --falkordb or --falkordb-push flag) diff --git a/tools/skillgen/expected/graphify__skills__claude__references__extraction-spec.md b/tools/skillgen/expected/graphify__skills__claude__references__extraction-spec.md index a5a12a575e..388df7674f 100644 --- a/tools/skillgen/expected/graphify__skills__claude__references__extraction-spec.md +++ b/tools/skillgen/expected/graphify__skills__claude__references__extraction-spec.md @@ -61,7 +61,7 @@ confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a d Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is the **full repo-relative path with the extension dropped**, every path segment kept and joined with `_` (each segment lowercased with non-alphanumeric chars replaced by `_`), and entity is the symbol name similarly normalized. Use every directory level, not just the immediate parent — this keeps same-named files in different directories distinct. Examples: `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `lib_utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`; `docs/v1/api/README.md` + `getUser` → `docs_v1_api_readme_getuser`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or only the immediate parent (e.g., `auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project built under the old immediate-parent format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. Generate the extraction JSON matching this schema exactly: -{"nodes":[{"id":"src_auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} +{"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. diff --git a/tools/skillgen/expected/graphify__skills__claude__references__query.md b/tools/skillgen/expected/graphify__skills__claude__references__query.md index 6dd627863c..56565eb782 100644 --- a/tools/skillgen/expected/graphify__skills__claude__references__query.md +++ b/tools/skillgen/expected/graphify__skills__claude__references__query.md @@ -79,12 +79,12 @@ If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal ```bash $(cat graphify-out/.graphify_python) -c " import sys, json -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph import networkx as nx from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' @@ -197,11 +197,11 @@ If the CLI is unavailable, run it inline: $(cat graphify-out/.graphify_python) -c " import json, sys import networkx as nx -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') a_term = 'NODE_A' b_term = 'NODE_B' @@ -265,11 +265,11 @@ If the CLI is unavailable, run it inline: $(cat graphify-out/.graphify_python) -c " import json, sys import networkx as nx -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') term = 'NODE_NAME' term_lower = term.lower() diff --git a/tools/skillgen/expected/graphify__skills__claude__references__transcribe.md b/tools/skillgen/expected/graphify__skills__claude__references__transcribe.md index bbe8b93e45..b967f83799 100644 --- a/tools/skillgen/expected/graphify__skills__claude__references__transcribe.md +++ b/tools/skillgen/expected/graphify__skills__claude__references__transcribe.md @@ -8,11 +8,13 @@ Skip this step entirely if `detect` returned zero `video` files. Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. -**Strategy:** If `graphify-out/.graphify_analysis.json` exists from a previous run, read its top god-node labels and write a one-sentence domain hint from them. Otherwise use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` Pass the chosen prompt to Whisper as its initial prompt. No separate API call needed. +**Strategy:** Read the god nodes from `graphify-out/.graphify_detect.json` (or the analysis file if it exists from a previous run). You are already a language model — write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. + +**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` **Step 1 - Write the Whisper prompt yourself.** -Read the top god-node labels from previous analysis when available, then compose a short domain hint sentence. With no previous analysis, use the generic fallback above. +Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: - Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` - Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` diff --git a/tools/skillgen/expected/graphify__skills__claude__references__update.md b/tools/skillgen/expected/graphify__skills__claude__references__update.md index 058ca8e6d2..3632fd4126 100644 --- a/tools/skillgen/expected/graphify__skills__claude__references__update.md +++ b/tools/skillgen/expected/graphify__skills__claude__references__update.md @@ -12,7 +12,7 @@ import sys, json from graphify.detect import detect_incremental, save_manifest from pathlib import Path -result = detect_incremental(Path(sys.argv[1])) +result = detect_incremental(Path('INPUT_PATH')) new_total = result.get('new_total', 0) print(json.dumps(result, indent=2, ensure_ascii=False)) Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") @@ -24,7 +24,7 @@ if deleted: print(f'{len(deleted)} deleted file(s) to prune.') if new_total > 0: print(f'{new_total} new/changed file(s) to re-extract.') -" "INPUT_PATH" +" ``` Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: @@ -84,7 +84,7 @@ Then: ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from pathlib import Path from graphify.build import build_merge from graphify.detect import save_manifest @@ -113,7 +113,7 @@ G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, - root=sys.argv[1], + root='INPUT_PATH', directed=IS_DIRECTED, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -150,7 +150,7 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) # is lost forever (#2015). Mirrors the library extract path # (cli._stamped_manifest_files + clear_semantic + scan_corpus). from graphify.cli import _stamped_manifest_files -_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path(sys.argv[1])) +_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) # Changed semantic files dispatched this run but NOT stamped had their chunk fail # or be omitted; clear any stale semantic_hash so they are re-queued (#1948). _sem_types = ('document', 'paper', 'image') @@ -160,9 +160,9 @@ _cleared = _dispatched - _stamped # scan_corpus = the RAW full corpus so in-root files newly excluded since last run # are dropped rather than masquerading as deletions; untouched rows preserved (#1908). _scan = {f for fl in incremental['files'].values() for f in fl} -save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) print('[graphify update] Manifest saved.') -" "INPUT_PATH" +" ``` Then run Steps 4–8 on the merged graph as normal. @@ -174,7 +174,7 @@ $(cat graphify-out/.graphify_python) -c " import json from graphify.analyze import graph_diff from graphify.build import build_from_json -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph import networkx as nx from pathlib import Path @@ -184,7 +184,7 @@ new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(e G_new = build_from_json(new_extract, directed=IS_DIRECTED) if old_data: - G_old = load_node_link_graph(old_data) + G_old = json_graph.node_link_graph(old_data, edges='links') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: diff --git a/tools/skillgen/expected/graphify__skills__claw__references__add-watch.md b/tools/skillgen/expected/graphify__skills__claw__references__add-watch.md index fcbe0cd488..77844343e1 100644 --- a/tools/skillgen/expected/graphify__skills__claw__references__add-watch.md +++ b/tools/skillgen/expected/graphify__skills__claw__references__add-watch.md @@ -13,7 +13,7 @@ from graphify.ingest import ingest from pathlib import Path try: - out = ingest(sys.argv[1], Path('./raw'), author=sys.argv[2] or None, contributor=sys.argv[3] or None) + out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') print(f'Saved to {out}') except ValueError as e: print(f'error: {e}', file=sys.stderr) @@ -21,10 +21,10 @@ except ValueError as e: except RuntimeError as e: print(f'error: {e}', file=sys.stderr) sys.exit(1) -" "URL" "AUTHOR" "CONTRIBUTOR" +" ``` -Replace `URL` with the actual URL and pass the user's `AUTHOR` or `CONTRIBUTOR` when provided; use an empty string for either omitted value. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on the scan root recorded in `graphify-out/.graphify_root` to merge the new file into the existing graph. +Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. Supported URL types (auto-detected): - YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`) @@ -41,7 +41,7 @@ Supported URL types (auto-detected): Start a background watcher that monitors a folder and auto-updates the graph when files change. ```bash -$(cat graphify-out/.graphify_python) -m graphify.watch "INPUT_PATH" --debounce 3 +$(cat graphify-out/.graphify_python) -m graphify.watch INPUT_PATH --debounce 3 ``` Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: diff --git a/tools/skillgen/expected/graphify__skills__claw__references__exports.md b/tools/skillgen/expected/graphify__skills__claw__references__exports.md index 89ce996fb7..242ff868e0 100644 --- a/tools/skillgen/expected/graphify__skills__claw__references__exports.md +++ b/tools/skillgen/expected/graphify__skills__claw__references__exports.md @@ -20,27 +20,13 @@ graphify export wiki graphify export neo4j ``` -**If `--neo4j-push `** - push directly to a running Neo4j instance. Do not run these blocks through an agent tool: its shell is non-interactive and cannot read a hidden password. Show the block for the user's shell, ask them to run it in their own terminal, and wait for confirmation before continuing. Never ask them to paste the password into chat. - -POSIX shell: +**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: ```bash -NEO4J_PASSWORD="$("$(cat graphify-out/.graphify_python)" -c 'import getpass; print(getpass.getpass("Neo4j password: "))')" -export NEO4J_PASSWORD -graphify export neo4j --push bolt://localhost:7687 --user neo4j -unset NEO4J_PASSWORD -``` - -PowerShell: - -```powershell -$credential = Read-Host "Neo4j password" -AsSecureString -$env:NEO4J_PASSWORD = [System.Net.NetworkCredential]::new("", $credential).Password -graphify export neo4j --push bolt://localhost:7687 --user neo4j -Remove-Item Env:NEO4J_PASSWORD +graphify export neo4j --push bolt://localhost:7687 --user neo4j --password PASSWORD ``` -Default URI is `bolt://localhost:7687`, default user is `neo4j`. The command reads the password from `NEO4J_PASSWORD`. Uses MERGE - safe to re-run without creating duplicates. +Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. ### Step 7a - FalkorDB export (only if --falkordb or --falkordb-push flag) diff --git a/tools/skillgen/expected/graphify__skills__claw__references__extraction-spec.md b/tools/skillgen/expected/graphify__skills__claw__references__extraction-spec.md index 8aaa23729b..4b278b28d3 100644 --- a/tools/skillgen/expected/graphify__skills__claw__references__extraction-spec.md +++ b/tools/skillgen/expected/graphify__skills__claw__references__extraction-spec.md @@ -25,7 +25,7 @@ Rules: Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format `{stem}_{entity}` where stem is the full repo-relative path with the extension dropped, every segment joined with `_` (each lowercased with non-alphanumeric chars replaced by `_`) and entity is the symbol name similarly normalized. Use every directory level, not just the immediate parent. `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`. Top-level files use just the filename stem. This must match the AST extractor's ID. Never append chunk or sequence suffixes — IDs must be deterministic from the label alone. Output exactly this JSON (no other text): -{"nodes":[{"id":"src_auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} +{"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} source_file RULE: set source_file to the FILE_LIST path for that file VERBATIM (absolute, no shortening to basename, no re-relativizing, no separator change). Keeps full build and --update on one base so build_merge's replace matches instead of duplicating. ``` diff --git a/tools/skillgen/expected/graphify__skills__claw__references__query.md b/tools/skillgen/expected/graphify__skills__claw__references__query.md index 6dd627863c..56565eb782 100644 --- a/tools/skillgen/expected/graphify__skills__claw__references__query.md +++ b/tools/skillgen/expected/graphify__skills__claw__references__query.md @@ -79,12 +79,12 @@ If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal ```bash $(cat graphify-out/.graphify_python) -c " import sys, json -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph import networkx as nx from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' @@ -197,11 +197,11 @@ If the CLI is unavailable, run it inline: $(cat graphify-out/.graphify_python) -c " import json, sys import networkx as nx -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') a_term = 'NODE_A' b_term = 'NODE_B' @@ -265,11 +265,11 @@ If the CLI is unavailable, run it inline: $(cat graphify-out/.graphify_python) -c " import json, sys import networkx as nx -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') term = 'NODE_NAME' term_lower = term.lower() diff --git a/tools/skillgen/expected/graphify__skills__claw__references__transcribe.md b/tools/skillgen/expected/graphify__skills__claw__references__transcribe.md index bbe8b93e45..b967f83799 100644 --- a/tools/skillgen/expected/graphify__skills__claw__references__transcribe.md +++ b/tools/skillgen/expected/graphify__skills__claw__references__transcribe.md @@ -8,11 +8,13 @@ Skip this step entirely if `detect` returned zero `video` files. Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. -**Strategy:** If `graphify-out/.graphify_analysis.json` exists from a previous run, read its top god-node labels and write a one-sentence domain hint from them. Otherwise use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` Pass the chosen prompt to Whisper as its initial prompt. No separate API call needed. +**Strategy:** Read the god nodes from `graphify-out/.graphify_detect.json` (or the analysis file if it exists from a previous run). You are already a language model — write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. + +**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` **Step 1 - Write the Whisper prompt yourself.** -Read the top god-node labels from previous analysis when available, then compose a short domain hint sentence. With no previous analysis, use the generic fallback above. +Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: - Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` - Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` diff --git a/tools/skillgen/expected/graphify__skills__claw__references__update.md b/tools/skillgen/expected/graphify__skills__claw__references__update.md index 058ca8e6d2..3632fd4126 100644 --- a/tools/skillgen/expected/graphify__skills__claw__references__update.md +++ b/tools/skillgen/expected/graphify__skills__claw__references__update.md @@ -12,7 +12,7 @@ import sys, json from graphify.detect import detect_incremental, save_manifest from pathlib import Path -result = detect_incremental(Path(sys.argv[1])) +result = detect_incremental(Path('INPUT_PATH')) new_total = result.get('new_total', 0) print(json.dumps(result, indent=2, ensure_ascii=False)) Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") @@ -24,7 +24,7 @@ if deleted: print(f'{len(deleted)} deleted file(s) to prune.') if new_total > 0: print(f'{new_total} new/changed file(s) to re-extract.') -" "INPUT_PATH" +" ``` Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: @@ -84,7 +84,7 @@ Then: ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from pathlib import Path from graphify.build import build_merge from graphify.detect import save_manifest @@ -113,7 +113,7 @@ G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, - root=sys.argv[1], + root='INPUT_PATH', directed=IS_DIRECTED, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -150,7 +150,7 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) # is lost forever (#2015). Mirrors the library extract path # (cli._stamped_manifest_files + clear_semantic + scan_corpus). from graphify.cli import _stamped_manifest_files -_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path(sys.argv[1])) +_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) # Changed semantic files dispatched this run but NOT stamped had their chunk fail # or be omitted; clear any stale semantic_hash so they are re-queued (#1948). _sem_types = ('document', 'paper', 'image') @@ -160,9 +160,9 @@ _cleared = _dispatched - _stamped # scan_corpus = the RAW full corpus so in-root files newly excluded since last run # are dropped rather than masquerading as deletions; untouched rows preserved (#1908). _scan = {f for fl in incremental['files'].values() for f in fl} -save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) print('[graphify update] Manifest saved.') -" "INPUT_PATH" +" ``` Then run Steps 4–8 on the merged graph as normal. @@ -174,7 +174,7 @@ $(cat graphify-out/.graphify_python) -c " import json from graphify.analyze import graph_diff from graphify.build import build_from_json -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph import networkx as nx from pathlib import Path @@ -184,7 +184,7 @@ new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(e G_new = build_from_json(new_extract, directed=IS_DIRECTED) if old_data: - G_old = load_node_link_graph(old_data) + G_old = json_graph.node_link_graph(old_data, edges='links') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: diff --git a/tools/skillgen/expected/graphify__skills__codex__references__add-watch.md b/tools/skillgen/expected/graphify__skills__codex__references__add-watch.md index fcbe0cd488..77844343e1 100644 --- a/tools/skillgen/expected/graphify__skills__codex__references__add-watch.md +++ b/tools/skillgen/expected/graphify__skills__codex__references__add-watch.md @@ -13,7 +13,7 @@ from graphify.ingest import ingest from pathlib import Path try: - out = ingest(sys.argv[1], Path('./raw'), author=sys.argv[2] or None, contributor=sys.argv[3] or None) + out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') print(f'Saved to {out}') except ValueError as e: print(f'error: {e}', file=sys.stderr) @@ -21,10 +21,10 @@ except ValueError as e: except RuntimeError as e: print(f'error: {e}', file=sys.stderr) sys.exit(1) -" "URL" "AUTHOR" "CONTRIBUTOR" +" ``` -Replace `URL` with the actual URL and pass the user's `AUTHOR` or `CONTRIBUTOR` when provided; use an empty string for either omitted value. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on the scan root recorded in `graphify-out/.graphify_root` to merge the new file into the existing graph. +Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. Supported URL types (auto-detected): - YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`) @@ -41,7 +41,7 @@ Supported URL types (auto-detected): Start a background watcher that monitors a folder and auto-updates the graph when files change. ```bash -$(cat graphify-out/.graphify_python) -m graphify.watch "INPUT_PATH" --debounce 3 +$(cat graphify-out/.graphify_python) -m graphify.watch INPUT_PATH --debounce 3 ``` Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: diff --git a/tools/skillgen/expected/graphify__skills__codex__references__exports.md b/tools/skillgen/expected/graphify__skills__codex__references__exports.md index 89ce996fb7..242ff868e0 100644 --- a/tools/skillgen/expected/graphify__skills__codex__references__exports.md +++ b/tools/skillgen/expected/graphify__skills__codex__references__exports.md @@ -20,27 +20,13 @@ graphify export wiki graphify export neo4j ``` -**If `--neo4j-push `** - push directly to a running Neo4j instance. Do not run these blocks through an agent tool: its shell is non-interactive and cannot read a hidden password. Show the block for the user's shell, ask them to run it in their own terminal, and wait for confirmation before continuing. Never ask them to paste the password into chat. - -POSIX shell: +**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: ```bash -NEO4J_PASSWORD="$("$(cat graphify-out/.graphify_python)" -c 'import getpass; print(getpass.getpass("Neo4j password: "))')" -export NEO4J_PASSWORD -graphify export neo4j --push bolt://localhost:7687 --user neo4j -unset NEO4J_PASSWORD -``` - -PowerShell: - -```powershell -$credential = Read-Host "Neo4j password" -AsSecureString -$env:NEO4J_PASSWORD = [System.Net.NetworkCredential]::new("", $credential).Password -graphify export neo4j --push bolt://localhost:7687 --user neo4j -Remove-Item Env:NEO4J_PASSWORD +graphify export neo4j --push bolt://localhost:7687 --user neo4j --password PASSWORD ``` -Default URI is `bolt://localhost:7687`, default user is `neo4j`. The command reads the password from `NEO4J_PASSWORD`. Uses MERGE - safe to re-run without creating duplicates. +Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. ### Step 7a - FalkorDB export (only if --falkordb or --falkordb-push flag) diff --git a/tools/skillgen/expected/graphify__skills__codex__references__extraction-spec.md b/tools/skillgen/expected/graphify__skills__codex__references__extraction-spec.md index 8aaa23729b..4b278b28d3 100644 --- a/tools/skillgen/expected/graphify__skills__codex__references__extraction-spec.md +++ b/tools/skillgen/expected/graphify__skills__codex__references__extraction-spec.md @@ -25,7 +25,7 @@ Rules: Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format `{stem}_{entity}` where stem is the full repo-relative path with the extension dropped, every segment joined with `_` (each lowercased with non-alphanumeric chars replaced by `_`) and entity is the symbol name similarly normalized. Use every directory level, not just the immediate parent. `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`. Top-level files use just the filename stem. This must match the AST extractor's ID. Never append chunk or sequence suffixes — IDs must be deterministic from the label alone. Output exactly this JSON (no other text): -{"nodes":[{"id":"src_auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} +{"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} source_file RULE: set source_file to the FILE_LIST path for that file VERBATIM (absolute, no shortening to basename, no re-relativizing, no separator change). Keeps full build and --update on one base so build_merge's replace matches instead of duplicating. ``` diff --git a/tools/skillgen/expected/graphify__skills__codex__references__query.md b/tools/skillgen/expected/graphify__skills__codex__references__query.md index 6dd627863c..56565eb782 100644 --- a/tools/skillgen/expected/graphify__skills__codex__references__query.md +++ b/tools/skillgen/expected/graphify__skills__codex__references__query.md @@ -79,12 +79,12 @@ If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal ```bash $(cat graphify-out/.graphify_python) -c " import sys, json -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph import networkx as nx from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' @@ -197,11 +197,11 @@ If the CLI is unavailable, run it inline: $(cat graphify-out/.graphify_python) -c " import json, sys import networkx as nx -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') a_term = 'NODE_A' b_term = 'NODE_B' @@ -265,11 +265,11 @@ If the CLI is unavailable, run it inline: $(cat graphify-out/.graphify_python) -c " import json, sys import networkx as nx -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') term = 'NODE_NAME' term_lower = term.lower() diff --git a/tools/skillgen/expected/graphify__skills__codex__references__transcribe.md b/tools/skillgen/expected/graphify__skills__codex__references__transcribe.md index bbe8b93e45..b967f83799 100644 --- a/tools/skillgen/expected/graphify__skills__codex__references__transcribe.md +++ b/tools/skillgen/expected/graphify__skills__codex__references__transcribe.md @@ -8,11 +8,13 @@ Skip this step entirely if `detect` returned zero `video` files. Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. -**Strategy:** If `graphify-out/.graphify_analysis.json` exists from a previous run, read its top god-node labels and write a one-sentence domain hint from them. Otherwise use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` Pass the chosen prompt to Whisper as its initial prompt. No separate API call needed. +**Strategy:** Read the god nodes from `graphify-out/.graphify_detect.json` (or the analysis file if it exists from a previous run). You are already a language model — write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. + +**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` **Step 1 - Write the Whisper prompt yourself.** -Read the top god-node labels from previous analysis when available, then compose a short domain hint sentence. With no previous analysis, use the generic fallback above. +Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: - Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` - Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` diff --git a/tools/skillgen/expected/graphify__skills__codex__references__update.md b/tools/skillgen/expected/graphify__skills__codex__references__update.md index 058ca8e6d2..3632fd4126 100644 --- a/tools/skillgen/expected/graphify__skills__codex__references__update.md +++ b/tools/skillgen/expected/graphify__skills__codex__references__update.md @@ -12,7 +12,7 @@ import sys, json from graphify.detect import detect_incremental, save_manifest from pathlib import Path -result = detect_incremental(Path(sys.argv[1])) +result = detect_incremental(Path('INPUT_PATH')) new_total = result.get('new_total', 0) print(json.dumps(result, indent=2, ensure_ascii=False)) Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") @@ -24,7 +24,7 @@ if deleted: print(f'{len(deleted)} deleted file(s) to prune.') if new_total > 0: print(f'{new_total} new/changed file(s) to re-extract.') -" "INPUT_PATH" +" ``` Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: @@ -84,7 +84,7 @@ Then: ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from pathlib import Path from graphify.build import build_merge from graphify.detect import save_manifest @@ -113,7 +113,7 @@ G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, - root=sys.argv[1], + root='INPUT_PATH', directed=IS_DIRECTED, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -150,7 +150,7 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) # is lost forever (#2015). Mirrors the library extract path # (cli._stamped_manifest_files + clear_semantic + scan_corpus). from graphify.cli import _stamped_manifest_files -_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path(sys.argv[1])) +_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) # Changed semantic files dispatched this run but NOT stamped had their chunk fail # or be omitted; clear any stale semantic_hash so they are re-queued (#1948). _sem_types = ('document', 'paper', 'image') @@ -160,9 +160,9 @@ _cleared = _dispatched - _stamped # scan_corpus = the RAW full corpus so in-root files newly excluded since last run # are dropped rather than masquerading as deletions; untouched rows preserved (#1908). _scan = {f for fl in incremental['files'].values() for f in fl} -save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) print('[graphify update] Manifest saved.') -" "INPUT_PATH" +" ``` Then run Steps 4–8 on the merged graph as normal. @@ -174,7 +174,7 @@ $(cat graphify-out/.graphify_python) -c " import json from graphify.analyze import graph_diff from graphify.build import build_from_json -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph import networkx as nx from pathlib import Path @@ -184,7 +184,7 @@ new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(e G_new = build_from_json(new_extract, directed=IS_DIRECTED) if old_data: - G_old = load_node_link_graph(old_data) + G_old = json_graph.node_link_graph(old_data, edges='links') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: diff --git a/tools/skillgen/expected/graphify__skills__copilot__references__add-watch.md b/tools/skillgen/expected/graphify__skills__copilot__references__add-watch.md index fcbe0cd488..77844343e1 100644 --- a/tools/skillgen/expected/graphify__skills__copilot__references__add-watch.md +++ b/tools/skillgen/expected/graphify__skills__copilot__references__add-watch.md @@ -13,7 +13,7 @@ from graphify.ingest import ingest from pathlib import Path try: - out = ingest(sys.argv[1], Path('./raw'), author=sys.argv[2] or None, contributor=sys.argv[3] or None) + out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') print(f'Saved to {out}') except ValueError as e: print(f'error: {e}', file=sys.stderr) @@ -21,10 +21,10 @@ except ValueError as e: except RuntimeError as e: print(f'error: {e}', file=sys.stderr) sys.exit(1) -" "URL" "AUTHOR" "CONTRIBUTOR" +" ``` -Replace `URL` with the actual URL and pass the user's `AUTHOR` or `CONTRIBUTOR` when provided; use an empty string for either omitted value. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on the scan root recorded in `graphify-out/.graphify_root` to merge the new file into the existing graph. +Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. Supported URL types (auto-detected): - YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`) @@ -41,7 +41,7 @@ Supported URL types (auto-detected): Start a background watcher that monitors a folder and auto-updates the graph when files change. ```bash -$(cat graphify-out/.graphify_python) -m graphify.watch "INPUT_PATH" --debounce 3 +$(cat graphify-out/.graphify_python) -m graphify.watch INPUT_PATH --debounce 3 ``` Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: diff --git a/tools/skillgen/expected/graphify__skills__copilot__references__exports.md b/tools/skillgen/expected/graphify__skills__copilot__references__exports.md index 89ce996fb7..242ff868e0 100644 --- a/tools/skillgen/expected/graphify__skills__copilot__references__exports.md +++ b/tools/skillgen/expected/graphify__skills__copilot__references__exports.md @@ -20,27 +20,13 @@ graphify export wiki graphify export neo4j ``` -**If `--neo4j-push `** - push directly to a running Neo4j instance. Do not run these blocks through an agent tool: its shell is non-interactive and cannot read a hidden password. Show the block for the user's shell, ask them to run it in their own terminal, and wait for confirmation before continuing. Never ask them to paste the password into chat. - -POSIX shell: +**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: ```bash -NEO4J_PASSWORD="$("$(cat graphify-out/.graphify_python)" -c 'import getpass; print(getpass.getpass("Neo4j password: "))')" -export NEO4J_PASSWORD -graphify export neo4j --push bolt://localhost:7687 --user neo4j -unset NEO4J_PASSWORD -``` - -PowerShell: - -```powershell -$credential = Read-Host "Neo4j password" -AsSecureString -$env:NEO4J_PASSWORD = [System.Net.NetworkCredential]::new("", $credential).Password -graphify export neo4j --push bolt://localhost:7687 --user neo4j -Remove-Item Env:NEO4J_PASSWORD +graphify export neo4j --push bolt://localhost:7687 --user neo4j --password PASSWORD ``` -Default URI is `bolt://localhost:7687`, default user is `neo4j`. The command reads the password from `NEO4J_PASSWORD`. Uses MERGE - safe to re-run without creating duplicates. +Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. ### Step 7a - FalkorDB export (only if --falkordb or --falkordb-push flag) diff --git a/tools/skillgen/expected/graphify__skills__copilot__references__extraction-spec.md b/tools/skillgen/expected/graphify__skills__copilot__references__extraction-spec.md index a5a12a575e..388df7674f 100644 --- a/tools/skillgen/expected/graphify__skills__copilot__references__extraction-spec.md +++ b/tools/skillgen/expected/graphify__skills__copilot__references__extraction-spec.md @@ -61,7 +61,7 @@ confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a d Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is the **full repo-relative path with the extension dropped**, every path segment kept and joined with `_` (each segment lowercased with non-alphanumeric chars replaced by `_`), and entity is the symbol name similarly normalized. Use every directory level, not just the immediate parent — this keeps same-named files in different directories distinct. Examples: `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `lib_utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`; `docs/v1/api/README.md` + `getUser` → `docs_v1_api_readme_getuser`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or only the immediate parent (e.g., `auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project built under the old immediate-parent format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. Generate the extraction JSON matching this schema exactly: -{"nodes":[{"id":"src_auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} +{"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. diff --git a/tools/skillgen/expected/graphify__skills__copilot__references__query.md b/tools/skillgen/expected/graphify__skills__copilot__references__query.md index 6dd627863c..56565eb782 100644 --- a/tools/skillgen/expected/graphify__skills__copilot__references__query.md +++ b/tools/skillgen/expected/graphify__skills__copilot__references__query.md @@ -79,12 +79,12 @@ If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal ```bash $(cat graphify-out/.graphify_python) -c " import sys, json -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph import networkx as nx from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' @@ -197,11 +197,11 @@ If the CLI is unavailable, run it inline: $(cat graphify-out/.graphify_python) -c " import json, sys import networkx as nx -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') a_term = 'NODE_A' b_term = 'NODE_B' @@ -265,11 +265,11 @@ If the CLI is unavailable, run it inline: $(cat graphify-out/.graphify_python) -c " import json, sys import networkx as nx -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') term = 'NODE_NAME' term_lower = term.lower() diff --git a/tools/skillgen/expected/graphify__skills__copilot__references__transcribe.md b/tools/skillgen/expected/graphify__skills__copilot__references__transcribe.md index bbe8b93e45..b967f83799 100644 --- a/tools/skillgen/expected/graphify__skills__copilot__references__transcribe.md +++ b/tools/skillgen/expected/graphify__skills__copilot__references__transcribe.md @@ -8,11 +8,13 @@ Skip this step entirely if `detect` returned zero `video` files. Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. -**Strategy:** If `graphify-out/.graphify_analysis.json` exists from a previous run, read its top god-node labels and write a one-sentence domain hint from them. Otherwise use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` Pass the chosen prompt to Whisper as its initial prompt. No separate API call needed. +**Strategy:** Read the god nodes from `graphify-out/.graphify_detect.json` (or the analysis file if it exists from a previous run). You are already a language model — write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. + +**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` **Step 1 - Write the Whisper prompt yourself.** -Read the top god-node labels from previous analysis when available, then compose a short domain hint sentence. With no previous analysis, use the generic fallback above. +Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: - Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` - Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` diff --git a/tools/skillgen/expected/graphify__skills__copilot__references__update.md b/tools/skillgen/expected/graphify__skills__copilot__references__update.md index 058ca8e6d2..3632fd4126 100644 --- a/tools/skillgen/expected/graphify__skills__copilot__references__update.md +++ b/tools/skillgen/expected/graphify__skills__copilot__references__update.md @@ -12,7 +12,7 @@ import sys, json from graphify.detect import detect_incremental, save_manifest from pathlib import Path -result = detect_incremental(Path(sys.argv[1])) +result = detect_incremental(Path('INPUT_PATH')) new_total = result.get('new_total', 0) print(json.dumps(result, indent=2, ensure_ascii=False)) Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") @@ -24,7 +24,7 @@ if deleted: print(f'{len(deleted)} deleted file(s) to prune.') if new_total > 0: print(f'{new_total} new/changed file(s) to re-extract.') -" "INPUT_PATH" +" ``` Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: @@ -84,7 +84,7 @@ Then: ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from pathlib import Path from graphify.build import build_merge from graphify.detect import save_manifest @@ -113,7 +113,7 @@ G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, - root=sys.argv[1], + root='INPUT_PATH', directed=IS_DIRECTED, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -150,7 +150,7 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) # is lost forever (#2015). Mirrors the library extract path # (cli._stamped_manifest_files + clear_semantic + scan_corpus). from graphify.cli import _stamped_manifest_files -_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path(sys.argv[1])) +_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) # Changed semantic files dispatched this run but NOT stamped had their chunk fail # or be omitted; clear any stale semantic_hash so they are re-queued (#1948). _sem_types = ('document', 'paper', 'image') @@ -160,9 +160,9 @@ _cleared = _dispatched - _stamped # scan_corpus = the RAW full corpus so in-root files newly excluded since last run # are dropped rather than masquerading as deletions; untouched rows preserved (#1908). _scan = {f for fl in incremental['files'].values() for f in fl} -save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) print('[graphify update] Manifest saved.') -" "INPUT_PATH" +" ``` Then run Steps 4–8 on the merged graph as normal. @@ -174,7 +174,7 @@ $(cat graphify-out/.graphify_python) -c " import json from graphify.analyze import graph_diff from graphify.build import build_from_json -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph import networkx as nx from pathlib import Path @@ -184,7 +184,7 @@ new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(e G_new = build_from_json(new_extract, directed=IS_DIRECTED) if old_data: - G_old = load_node_link_graph(old_data) + G_old = json_graph.node_link_graph(old_data, edges='links') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: diff --git a/tools/skillgen/expected/graphify__skills__droid__references__add-watch.md b/tools/skillgen/expected/graphify__skills__droid__references__add-watch.md index fcbe0cd488..77844343e1 100644 --- a/tools/skillgen/expected/graphify__skills__droid__references__add-watch.md +++ b/tools/skillgen/expected/graphify__skills__droid__references__add-watch.md @@ -13,7 +13,7 @@ from graphify.ingest import ingest from pathlib import Path try: - out = ingest(sys.argv[1], Path('./raw'), author=sys.argv[2] or None, contributor=sys.argv[3] or None) + out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') print(f'Saved to {out}') except ValueError as e: print(f'error: {e}', file=sys.stderr) @@ -21,10 +21,10 @@ except ValueError as e: except RuntimeError as e: print(f'error: {e}', file=sys.stderr) sys.exit(1) -" "URL" "AUTHOR" "CONTRIBUTOR" +" ``` -Replace `URL` with the actual URL and pass the user's `AUTHOR` or `CONTRIBUTOR` when provided; use an empty string for either omitted value. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on the scan root recorded in `graphify-out/.graphify_root` to merge the new file into the existing graph. +Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. Supported URL types (auto-detected): - YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`) @@ -41,7 +41,7 @@ Supported URL types (auto-detected): Start a background watcher that monitors a folder and auto-updates the graph when files change. ```bash -$(cat graphify-out/.graphify_python) -m graphify.watch "INPUT_PATH" --debounce 3 +$(cat graphify-out/.graphify_python) -m graphify.watch INPUT_PATH --debounce 3 ``` Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: diff --git a/tools/skillgen/expected/graphify__skills__droid__references__exports.md b/tools/skillgen/expected/graphify__skills__droid__references__exports.md index 89ce996fb7..242ff868e0 100644 --- a/tools/skillgen/expected/graphify__skills__droid__references__exports.md +++ b/tools/skillgen/expected/graphify__skills__droid__references__exports.md @@ -20,27 +20,13 @@ graphify export wiki graphify export neo4j ``` -**If `--neo4j-push `** - push directly to a running Neo4j instance. Do not run these blocks through an agent tool: its shell is non-interactive and cannot read a hidden password. Show the block for the user's shell, ask them to run it in their own terminal, and wait for confirmation before continuing. Never ask them to paste the password into chat. - -POSIX shell: +**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: ```bash -NEO4J_PASSWORD="$("$(cat graphify-out/.graphify_python)" -c 'import getpass; print(getpass.getpass("Neo4j password: "))')" -export NEO4J_PASSWORD -graphify export neo4j --push bolt://localhost:7687 --user neo4j -unset NEO4J_PASSWORD -``` - -PowerShell: - -```powershell -$credential = Read-Host "Neo4j password" -AsSecureString -$env:NEO4J_PASSWORD = [System.Net.NetworkCredential]::new("", $credential).Password -graphify export neo4j --push bolt://localhost:7687 --user neo4j -Remove-Item Env:NEO4J_PASSWORD +graphify export neo4j --push bolt://localhost:7687 --user neo4j --password PASSWORD ``` -Default URI is `bolt://localhost:7687`, default user is `neo4j`. The command reads the password from `NEO4J_PASSWORD`. Uses MERGE - safe to re-run without creating duplicates. +Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. ### Step 7a - FalkorDB export (only if --falkordb or --falkordb-push flag) diff --git a/tools/skillgen/expected/graphify__skills__droid__references__extraction-spec.md b/tools/skillgen/expected/graphify__skills__droid__references__extraction-spec.md index a5a12a575e..388df7674f 100644 --- a/tools/skillgen/expected/graphify__skills__droid__references__extraction-spec.md +++ b/tools/skillgen/expected/graphify__skills__droid__references__extraction-spec.md @@ -61,7 +61,7 @@ confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a d Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is the **full repo-relative path with the extension dropped**, every path segment kept and joined with `_` (each segment lowercased with non-alphanumeric chars replaced by `_`), and entity is the symbol name similarly normalized. Use every directory level, not just the immediate parent — this keeps same-named files in different directories distinct. Examples: `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `lib_utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`; `docs/v1/api/README.md` + `getUser` → `docs_v1_api_readme_getuser`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or only the immediate parent (e.g., `auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project built under the old immediate-parent format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. Generate the extraction JSON matching this schema exactly: -{"nodes":[{"id":"src_auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} +{"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. diff --git a/tools/skillgen/expected/graphify__skills__droid__references__query.md b/tools/skillgen/expected/graphify__skills__droid__references__query.md index 6dd627863c..56565eb782 100644 --- a/tools/skillgen/expected/graphify__skills__droid__references__query.md +++ b/tools/skillgen/expected/graphify__skills__droid__references__query.md @@ -79,12 +79,12 @@ If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal ```bash $(cat graphify-out/.graphify_python) -c " import sys, json -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph import networkx as nx from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' @@ -197,11 +197,11 @@ If the CLI is unavailable, run it inline: $(cat graphify-out/.graphify_python) -c " import json, sys import networkx as nx -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') a_term = 'NODE_A' b_term = 'NODE_B' @@ -265,11 +265,11 @@ If the CLI is unavailable, run it inline: $(cat graphify-out/.graphify_python) -c " import json, sys import networkx as nx -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') term = 'NODE_NAME' term_lower = term.lower() diff --git a/tools/skillgen/expected/graphify__skills__droid__references__transcribe.md b/tools/skillgen/expected/graphify__skills__droid__references__transcribe.md index bbe8b93e45..b967f83799 100644 --- a/tools/skillgen/expected/graphify__skills__droid__references__transcribe.md +++ b/tools/skillgen/expected/graphify__skills__droid__references__transcribe.md @@ -8,11 +8,13 @@ Skip this step entirely if `detect` returned zero `video` files. Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. -**Strategy:** If `graphify-out/.graphify_analysis.json` exists from a previous run, read its top god-node labels and write a one-sentence domain hint from them. Otherwise use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` Pass the chosen prompt to Whisper as its initial prompt. No separate API call needed. +**Strategy:** Read the god nodes from `graphify-out/.graphify_detect.json` (or the analysis file if it exists from a previous run). You are already a language model — write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. + +**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` **Step 1 - Write the Whisper prompt yourself.** -Read the top god-node labels from previous analysis when available, then compose a short domain hint sentence. With no previous analysis, use the generic fallback above. +Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: - Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` - Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` diff --git a/tools/skillgen/expected/graphify__skills__droid__references__update.md b/tools/skillgen/expected/graphify__skills__droid__references__update.md index 058ca8e6d2..3632fd4126 100644 --- a/tools/skillgen/expected/graphify__skills__droid__references__update.md +++ b/tools/skillgen/expected/graphify__skills__droid__references__update.md @@ -12,7 +12,7 @@ import sys, json from graphify.detect import detect_incremental, save_manifest from pathlib import Path -result = detect_incremental(Path(sys.argv[1])) +result = detect_incremental(Path('INPUT_PATH')) new_total = result.get('new_total', 0) print(json.dumps(result, indent=2, ensure_ascii=False)) Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") @@ -24,7 +24,7 @@ if deleted: print(f'{len(deleted)} deleted file(s) to prune.') if new_total > 0: print(f'{new_total} new/changed file(s) to re-extract.') -" "INPUT_PATH" +" ``` Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: @@ -84,7 +84,7 @@ Then: ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from pathlib import Path from graphify.build import build_merge from graphify.detect import save_manifest @@ -113,7 +113,7 @@ G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, - root=sys.argv[1], + root='INPUT_PATH', directed=IS_DIRECTED, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -150,7 +150,7 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) # is lost forever (#2015). Mirrors the library extract path # (cli._stamped_manifest_files + clear_semantic + scan_corpus). from graphify.cli import _stamped_manifest_files -_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path(sys.argv[1])) +_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) # Changed semantic files dispatched this run but NOT stamped had their chunk fail # or be omitted; clear any stale semantic_hash so they are re-queued (#1948). _sem_types = ('document', 'paper', 'image') @@ -160,9 +160,9 @@ _cleared = _dispatched - _stamped # scan_corpus = the RAW full corpus so in-root files newly excluded since last run # are dropped rather than masquerading as deletions; untouched rows preserved (#1908). _scan = {f for fl in incremental['files'].values() for f in fl} -save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) print('[graphify update] Manifest saved.') -" "INPUT_PATH" +" ``` Then run Steps 4–8 on the merged graph as normal. @@ -174,7 +174,7 @@ $(cat graphify-out/.graphify_python) -c " import json from graphify.analyze import graph_diff from graphify.build import build_from_json -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph import networkx as nx from pathlib import Path @@ -184,7 +184,7 @@ new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(e G_new = build_from_json(new_extract, directed=IS_DIRECTED) if old_data: - G_old = load_node_link_graph(old_data) + G_old = json_graph.node_link_graph(old_data, edges='links') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: diff --git a/tools/skillgen/expected/graphify__skills__kilo__references__add-watch.md b/tools/skillgen/expected/graphify__skills__kilo__references__add-watch.md index fcbe0cd488..77844343e1 100644 --- a/tools/skillgen/expected/graphify__skills__kilo__references__add-watch.md +++ b/tools/skillgen/expected/graphify__skills__kilo__references__add-watch.md @@ -13,7 +13,7 @@ from graphify.ingest import ingest from pathlib import Path try: - out = ingest(sys.argv[1], Path('./raw'), author=sys.argv[2] or None, contributor=sys.argv[3] or None) + out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') print(f'Saved to {out}') except ValueError as e: print(f'error: {e}', file=sys.stderr) @@ -21,10 +21,10 @@ except ValueError as e: except RuntimeError as e: print(f'error: {e}', file=sys.stderr) sys.exit(1) -" "URL" "AUTHOR" "CONTRIBUTOR" +" ``` -Replace `URL` with the actual URL and pass the user's `AUTHOR` or `CONTRIBUTOR` when provided; use an empty string for either omitted value. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on the scan root recorded in `graphify-out/.graphify_root` to merge the new file into the existing graph. +Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. Supported URL types (auto-detected): - YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`) @@ -41,7 +41,7 @@ Supported URL types (auto-detected): Start a background watcher that monitors a folder and auto-updates the graph when files change. ```bash -$(cat graphify-out/.graphify_python) -m graphify.watch "INPUT_PATH" --debounce 3 +$(cat graphify-out/.graphify_python) -m graphify.watch INPUT_PATH --debounce 3 ``` Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: diff --git a/tools/skillgen/expected/graphify__skills__kilo__references__exports.md b/tools/skillgen/expected/graphify__skills__kilo__references__exports.md index 89ce996fb7..242ff868e0 100644 --- a/tools/skillgen/expected/graphify__skills__kilo__references__exports.md +++ b/tools/skillgen/expected/graphify__skills__kilo__references__exports.md @@ -20,27 +20,13 @@ graphify export wiki graphify export neo4j ``` -**If `--neo4j-push `** - push directly to a running Neo4j instance. Do not run these blocks through an agent tool: its shell is non-interactive and cannot read a hidden password. Show the block for the user's shell, ask them to run it in their own terminal, and wait for confirmation before continuing. Never ask them to paste the password into chat. - -POSIX shell: +**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: ```bash -NEO4J_PASSWORD="$("$(cat graphify-out/.graphify_python)" -c 'import getpass; print(getpass.getpass("Neo4j password: "))')" -export NEO4J_PASSWORD -graphify export neo4j --push bolt://localhost:7687 --user neo4j -unset NEO4J_PASSWORD -``` - -PowerShell: - -```powershell -$credential = Read-Host "Neo4j password" -AsSecureString -$env:NEO4J_PASSWORD = [System.Net.NetworkCredential]::new("", $credential).Password -graphify export neo4j --push bolt://localhost:7687 --user neo4j -Remove-Item Env:NEO4J_PASSWORD +graphify export neo4j --push bolt://localhost:7687 --user neo4j --password PASSWORD ``` -Default URI is `bolt://localhost:7687`, default user is `neo4j`. The command reads the password from `NEO4J_PASSWORD`. Uses MERGE - safe to re-run without creating duplicates. +Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. ### Step 7a - FalkorDB export (only if --falkordb or --falkordb-push flag) diff --git a/tools/skillgen/expected/graphify__skills__kilo__references__extraction-spec.md b/tools/skillgen/expected/graphify__skills__kilo__references__extraction-spec.md index a5a12a575e..388df7674f 100644 --- a/tools/skillgen/expected/graphify__skills__kilo__references__extraction-spec.md +++ b/tools/skillgen/expected/graphify__skills__kilo__references__extraction-spec.md @@ -61,7 +61,7 @@ confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a d Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is the **full repo-relative path with the extension dropped**, every path segment kept and joined with `_` (each segment lowercased with non-alphanumeric chars replaced by `_`), and entity is the symbol name similarly normalized. Use every directory level, not just the immediate parent — this keeps same-named files in different directories distinct. Examples: `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `lib_utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`; `docs/v1/api/README.md` + `getUser` → `docs_v1_api_readme_getuser`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or only the immediate parent (e.g., `auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project built under the old immediate-parent format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. Generate the extraction JSON matching this schema exactly: -{"nodes":[{"id":"src_auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} +{"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. diff --git a/tools/skillgen/expected/graphify__skills__kilo__references__query.md b/tools/skillgen/expected/graphify__skills__kilo__references__query.md index 6dd627863c..56565eb782 100644 --- a/tools/skillgen/expected/graphify__skills__kilo__references__query.md +++ b/tools/skillgen/expected/graphify__skills__kilo__references__query.md @@ -79,12 +79,12 @@ If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal ```bash $(cat graphify-out/.graphify_python) -c " import sys, json -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph import networkx as nx from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' @@ -197,11 +197,11 @@ If the CLI is unavailable, run it inline: $(cat graphify-out/.graphify_python) -c " import json, sys import networkx as nx -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') a_term = 'NODE_A' b_term = 'NODE_B' @@ -265,11 +265,11 @@ If the CLI is unavailable, run it inline: $(cat graphify-out/.graphify_python) -c " import json, sys import networkx as nx -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') term = 'NODE_NAME' term_lower = term.lower() diff --git a/tools/skillgen/expected/graphify__skills__kilo__references__transcribe.md b/tools/skillgen/expected/graphify__skills__kilo__references__transcribe.md index bbe8b93e45..b967f83799 100644 --- a/tools/skillgen/expected/graphify__skills__kilo__references__transcribe.md +++ b/tools/skillgen/expected/graphify__skills__kilo__references__transcribe.md @@ -8,11 +8,13 @@ Skip this step entirely if `detect` returned zero `video` files. Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. -**Strategy:** If `graphify-out/.graphify_analysis.json` exists from a previous run, read its top god-node labels and write a one-sentence domain hint from them. Otherwise use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` Pass the chosen prompt to Whisper as its initial prompt. No separate API call needed. +**Strategy:** Read the god nodes from `graphify-out/.graphify_detect.json` (or the analysis file if it exists from a previous run). You are already a language model — write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. + +**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` **Step 1 - Write the Whisper prompt yourself.** -Read the top god-node labels from previous analysis when available, then compose a short domain hint sentence. With no previous analysis, use the generic fallback above. +Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: - Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` - Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` diff --git a/tools/skillgen/expected/graphify__skills__kilo__references__update.md b/tools/skillgen/expected/graphify__skills__kilo__references__update.md index 058ca8e6d2..3632fd4126 100644 --- a/tools/skillgen/expected/graphify__skills__kilo__references__update.md +++ b/tools/skillgen/expected/graphify__skills__kilo__references__update.md @@ -12,7 +12,7 @@ import sys, json from graphify.detect import detect_incremental, save_manifest from pathlib import Path -result = detect_incremental(Path(sys.argv[1])) +result = detect_incremental(Path('INPUT_PATH')) new_total = result.get('new_total', 0) print(json.dumps(result, indent=2, ensure_ascii=False)) Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") @@ -24,7 +24,7 @@ if deleted: print(f'{len(deleted)} deleted file(s) to prune.') if new_total > 0: print(f'{new_total} new/changed file(s) to re-extract.') -" "INPUT_PATH" +" ``` Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: @@ -84,7 +84,7 @@ Then: ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from pathlib import Path from graphify.build import build_merge from graphify.detect import save_manifest @@ -113,7 +113,7 @@ G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, - root=sys.argv[1], + root='INPUT_PATH', directed=IS_DIRECTED, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -150,7 +150,7 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) # is lost forever (#2015). Mirrors the library extract path # (cli._stamped_manifest_files + clear_semantic + scan_corpus). from graphify.cli import _stamped_manifest_files -_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path(sys.argv[1])) +_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) # Changed semantic files dispatched this run but NOT stamped had their chunk fail # or be omitted; clear any stale semantic_hash so they are re-queued (#1948). _sem_types = ('document', 'paper', 'image') @@ -160,9 +160,9 @@ _cleared = _dispatched - _stamped # scan_corpus = the RAW full corpus so in-root files newly excluded since last run # are dropped rather than masquerading as deletions; untouched rows preserved (#1908). _scan = {f for fl in incremental['files'].values() for f in fl} -save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) print('[graphify update] Manifest saved.') -" "INPUT_PATH" +" ``` Then run Steps 4–8 on the merged graph as normal. @@ -174,7 +174,7 @@ $(cat graphify-out/.graphify_python) -c " import json from graphify.analyze import graph_diff from graphify.build import build_from_json -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph import networkx as nx from pathlib import Path @@ -184,7 +184,7 @@ new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(e G_new = build_from_json(new_extract, directed=IS_DIRECTED) if old_data: - G_old = load_node_link_graph(old_data) + G_old = json_graph.node_link_graph(old_data, edges='links') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: diff --git a/tools/skillgen/expected/graphify__skills__kiro__references__add-watch.md b/tools/skillgen/expected/graphify__skills__kiro__references__add-watch.md index fcbe0cd488..77844343e1 100644 --- a/tools/skillgen/expected/graphify__skills__kiro__references__add-watch.md +++ b/tools/skillgen/expected/graphify__skills__kiro__references__add-watch.md @@ -13,7 +13,7 @@ from graphify.ingest import ingest from pathlib import Path try: - out = ingest(sys.argv[1], Path('./raw'), author=sys.argv[2] or None, contributor=sys.argv[3] or None) + out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') print(f'Saved to {out}') except ValueError as e: print(f'error: {e}', file=sys.stderr) @@ -21,10 +21,10 @@ except ValueError as e: except RuntimeError as e: print(f'error: {e}', file=sys.stderr) sys.exit(1) -" "URL" "AUTHOR" "CONTRIBUTOR" +" ``` -Replace `URL` with the actual URL and pass the user's `AUTHOR` or `CONTRIBUTOR` when provided; use an empty string for either omitted value. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on the scan root recorded in `graphify-out/.graphify_root` to merge the new file into the existing graph. +Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. Supported URL types (auto-detected): - YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`) @@ -41,7 +41,7 @@ Supported URL types (auto-detected): Start a background watcher that monitors a folder and auto-updates the graph when files change. ```bash -$(cat graphify-out/.graphify_python) -m graphify.watch "INPUT_PATH" --debounce 3 +$(cat graphify-out/.graphify_python) -m graphify.watch INPUT_PATH --debounce 3 ``` Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: diff --git a/tools/skillgen/expected/graphify__skills__kiro__references__exports.md b/tools/skillgen/expected/graphify__skills__kiro__references__exports.md index 89ce996fb7..242ff868e0 100644 --- a/tools/skillgen/expected/graphify__skills__kiro__references__exports.md +++ b/tools/skillgen/expected/graphify__skills__kiro__references__exports.md @@ -20,27 +20,13 @@ graphify export wiki graphify export neo4j ``` -**If `--neo4j-push `** - push directly to a running Neo4j instance. Do not run these blocks through an agent tool: its shell is non-interactive and cannot read a hidden password. Show the block for the user's shell, ask them to run it in their own terminal, and wait for confirmation before continuing. Never ask them to paste the password into chat. - -POSIX shell: +**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: ```bash -NEO4J_PASSWORD="$("$(cat graphify-out/.graphify_python)" -c 'import getpass; print(getpass.getpass("Neo4j password: "))')" -export NEO4J_PASSWORD -graphify export neo4j --push bolt://localhost:7687 --user neo4j -unset NEO4J_PASSWORD -``` - -PowerShell: - -```powershell -$credential = Read-Host "Neo4j password" -AsSecureString -$env:NEO4J_PASSWORD = [System.Net.NetworkCredential]::new("", $credential).Password -graphify export neo4j --push bolt://localhost:7687 --user neo4j -Remove-Item Env:NEO4J_PASSWORD +graphify export neo4j --push bolt://localhost:7687 --user neo4j --password PASSWORD ``` -Default URI is `bolt://localhost:7687`, default user is `neo4j`. The command reads the password from `NEO4J_PASSWORD`. Uses MERGE - safe to re-run without creating duplicates. +Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. ### Step 7a - FalkorDB export (only if --falkordb or --falkordb-push flag) diff --git a/tools/skillgen/expected/graphify__skills__kiro__references__extraction-spec.md b/tools/skillgen/expected/graphify__skills__kiro__references__extraction-spec.md index 8aaa23729b..4b278b28d3 100644 --- a/tools/skillgen/expected/graphify__skills__kiro__references__extraction-spec.md +++ b/tools/skillgen/expected/graphify__skills__kiro__references__extraction-spec.md @@ -25,7 +25,7 @@ Rules: Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format `{stem}_{entity}` where stem is the full repo-relative path with the extension dropped, every segment joined with `_` (each lowercased with non-alphanumeric chars replaced by `_`) and entity is the symbol name similarly normalized. Use every directory level, not just the immediate parent. `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`. Top-level files use just the filename stem. This must match the AST extractor's ID. Never append chunk or sequence suffixes — IDs must be deterministic from the label alone. Output exactly this JSON (no other text): -{"nodes":[{"id":"src_auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} +{"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} source_file RULE: set source_file to the FILE_LIST path for that file VERBATIM (absolute, no shortening to basename, no re-relativizing, no separator change). Keeps full build and --update on one base so build_merge's replace matches instead of duplicating. ``` diff --git a/tools/skillgen/expected/graphify__skills__kiro__references__query.md b/tools/skillgen/expected/graphify__skills__kiro__references__query.md index 6dd627863c..56565eb782 100644 --- a/tools/skillgen/expected/graphify__skills__kiro__references__query.md +++ b/tools/skillgen/expected/graphify__skills__kiro__references__query.md @@ -79,12 +79,12 @@ If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal ```bash $(cat graphify-out/.graphify_python) -c " import sys, json -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph import networkx as nx from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' @@ -197,11 +197,11 @@ If the CLI is unavailable, run it inline: $(cat graphify-out/.graphify_python) -c " import json, sys import networkx as nx -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') a_term = 'NODE_A' b_term = 'NODE_B' @@ -265,11 +265,11 @@ If the CLI is unavailable, run it inline: $(cat graphify-out/.graphify_python) -c " import json, sys import networkx as nx -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') term = 'NODE_NAME' term_lower = term.lower() diff --git a/tools/skillgen/expected/graphify__skills__kiro__references__transcribe.md b/tools/skillgen/expected/graphify__skills__kiro__references__transcribe.md index bbe8b93e45..b967f83799 100644 --- a/tools/skillgen/expected/graphify__skills__kiro__references__transcribe.md +++ b/tools/skillgen/expected/graphify__skills__kiro__references__transcribe.md @@ -8,11 +8,13 @@ Skip this step entirely if `detect` returned zero `video` files. Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. -**Strategy:** If `graphify-out/.graphify_analysis.json` exists from a previous run, read its top god-node labels and write a one-sentence domain hint from them. Otherwise use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` Pass the chosen prompt to Whisper as its initial prompt. No separate API call needed. +**Strategy:** Read the god nodes from `graphify-out/.graphify_detect.json` (or the analysis file if it exists from a previous run). You are already a language model — write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. + +**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` **Step 1 - Write the Whisper prompt yourself.** -Read the top god-node labels from previous analysis when available, then compose a short domain hint sentence. With no previous analysis, use the generic fallback above. +Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: - Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` - Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` diff --git a/tools/skillgen/expected/graphify__skills__kiro__references__update.md b/tools/skillgen/expected/graphify__skills__kiro__references__update.md index 058ca8e6d2..3632fd4126 100644 --- a/tools/skillgen/expected/graphify__skills__kiro__references__update.md +++ b/tools/skillgen/expected/graphify__skills__kiro__references__update.md @@ -12,7 +12,7 @@ import sys, json from graphify.detect import detect_incremental, save_manifest from pathlib import Path -result = detect_incremental(Path(sys.argv[1])) +result = detect_incremental(Path('INPUT_PATH')) new_total = result.get('new_total', 0) print(json.dumps(result, indent=2, ensure_ascii=False)) Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") @@ -24,7 +24,7 @@ if deleted: print(f'{len(deleted)} deleted file(s) to prune.') if new_total > 0: print(f'{new_total} new/changed file(s) to re-extract.') -" "INPUT_PATH" +" ``` Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: @@ -84,7 +84,7 @@ Then: ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from pathlib import Path from graphify.build import build_merge from graphify.detect import save_manifest @@ -113,7 +113,7 @@ G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, - root=sys.argv[1], + root='INPUT_PATH', directed=IS_DIRECTED, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -150,7 +150,7 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) # is lost forever (#2015). Mirrors the library extract path # (cli._stamped_manifest_files + clear_semantic + scan_corpus). from graphify.cli import _stamped_manifest_files -_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path(sys.argv[1])) +_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) # Changed semantic files dispatched this run but NOT stamped had their chunk fail # or be omitted; clear any stale semantic_hash so they are re-queued (#1948). _sem_types = ('document', 'paper', 'image') @@ -160,9 +160,9 @@ _cleared = _dispatched - _stamped # scan_corpus = the RAW full corpus so in-root files newly excluded since last run # are dropped rather than masquerading as deletions; untouched rows preserved (#1908). _scan = {f for fl in incremental['files'].values() for f in fl} -save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) print('[graphify update] Manifest saved.') -" "INPUT_PATH" +" ``` Then run Steps 4–8 on the merged graph as normal. @@ -174,7 +174,7 @@ $(cat graphify-out/.graphify_python) -c " import json from graphify.analyze import graph_diff from graphify.build import build_from_json -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph import networkx as nx from pathlib import Path @@ -184,7 +184,7 @@ new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(e G_new = build_from_json(new_extract, directed=IS_DIRECTED) if old_data: - G_old = load_node_link_graph(old_data) + G_old = json_graph.node_link_graph(old_data, edges='links') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: diff --git a/tools/skillgen/expected/graphify__skills__opencode__references__add-watch.md b/tools/skillgen/expected/graphify__skills__opencode__references__add-watch.md index fcbe0cd488..77844343e1 100644 --- a/tools/skillgen/expected/graphify__skills__opencode__references__add-watch.md +++ b/tools/skillgen/expected/graphify__skills__opencode__references__add-watch.md @@ -13,7 +13,7 @@ from graphify.ingest import ingest from pathlib import Path try: - out = ingest(sys.argv[1], Path('./raw'), author=sys.argv[2] or None, contributor=sys.argv[3] or None) + out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') print(f'Saved to {out}') except ValueError as e: print(f'error: {e}', file=sys.stderr) @@ -21,10 +21,10 @@ except ValueError as e: except RuntimeError as e: print(f'error: {e}', file=sys.stderr) sys.exit(1) -" "URL" "AUTHOR" "CONTRIBUTOR" +" ``` -Replace `URL` with the actual URL and pass the user's `AUTHOR` or `CONTRIBUTOR` when provided; use an empty string for either omitted value. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on the scan root recorded in `graphify-out/.graphify_root` to merge the new file into the existing graph. +Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. Supported URL types (auto-detected): - YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`) @@ -41,7 +41,7 @@ Supported URL types (auto-detected): Start a background watcher that monitors a folder and auto-updates the graph when files change. ```bash -$(cat graphify-out/.graphify_python) -m graphify.watch "INPUT_PATH" --debounce 3 +$(cat graphify-out/.graphify_python) -m graphify.watch INPUT_PATH --debounce 3 ``` Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: diff --git a/tools/skillgen/expected/graphify__skills__opencode__references__exports.md b/tools/skillgen/expected/graphify__skills__opencode__references__exports.md index 89ce996fb7..242ff868e0 100644 --- a/tools/skillgen/expected/graphify__skills__opencode__references__exports.md +++ b/tools/skillgen/expected/graphify__skills__opencode__references__exports.md @@ -20,27 +20,13 @@ graphify export wiki graphify export neo4j ``` -**If `--neo4j-push `** - push directly to a running Neo4j instance. Do not run these blocks through an agent tool: its shell is non-interactive and cannot read a hidden password. Show the block for the user's shell, ask them to run it in their own terminal, and wait for confirmation before continuing. Never ask them to paste the password into chat. - -POSIX shell: +**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: ```bash -NEO4J_PASSWORD="$("$(cat graphify-out/.graphify_python)" -c 'import getpass; print(getpass.getpass("Neo4j password: "))')" -export NEO4J_PASSWORD -graphify export neo4j --push bolt://localhost:7687 --user neo4j -unset NEO4J_PASSWORD -``` - -PowerShell: - -```powershell -$credential = Read-Host "Neo4j password" -AsSecureString -$env:NEO4J_PASSWORD = [System.Net.NetworkCredential]::new("", $credential).Password -graphify export neo4j --push bolt://localhost:7687 --user neo4j -Remove-Item Env:NEO4J_PASSWORD +graphify export neo4j --push bolt://localhost:7687 --user neo4j --password PASSWORD ``` -Default URI is `bolt://localhost:7687`, default user is `neo4j`. The command reads the password from `NEO4J_PASSWORD`. Uses MERGE - safe to re-run without creating duplicates. +Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. ### Step 7a - FalkorDB export (only if --falkordb or --falkordb-push flag) diff --git a/tools/skillgen/expected/graphify__skills__opencode__references__extraction-spec.md b/tools/skillgen/expected/graphify__skills__opencode__references__extraction-spec.md index a5a12a575e..388df7674f 100644 --- a/tools/skillgen/expected/graphify__skills__opencode__references__extraction-spec.md +++ b/tools/skillgen/expected/graphify__skills__opencode__references__extraction-spec.md @@ -61,7 +61,7 @@ confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a d Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is the **full repo-relative path with the extension dropped**, every path segment kept and joined with `_` (each segment lowercased with non-alphanumeric chars replaced by `_`), and entity is the symbol name similarly normalized. Use every directory level, not just the immediate parent — this keeps same-named files in different directories distinct. Examples: `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `lib_utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`; `docs/v1/api/README.md` + `getUser` → `docs_v1_api_readme_getuser`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or only the immediate parent (e.g., `auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project built under the old immediate-parent format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. Generate the extraction JSON matching this schema exactly: -{"nodes":[{"id":"src_auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} +{"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. diff --git a/tools/skillgen/expected/graphify__skills__opencode__references__query.md b/tools/skillgen/expected/graphify__skills__opencode__references__query.md index 6dd627863c..56565eb782 100644 --- a/tools/skillgen/expected/graphify__skills__opencode__references__query.md +++ b/tools/skillgen/expected/graphify__skills__opencode__references__query.md @@ -79,12 +79,12 @@ If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal ```bash $(cat graphify-out/.graphify_python) -c " import sys, json -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph import networkx as nx from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' @@ -197,11 +197,11 @@ If the CLI is unavailable, run it inline: $(cat graphify-out/.graphify_python) -c " import json, sys import networkx as nx -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') a_term = 'NODE_A' b_term = 'NODE_B' @@ -265,11 +265,11 @@ If the CLI is unavailable, run it inline: $(cat graphify-out/.graphify_python) -c " import json, sys import networkx as nx -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') term = 'NODE_NAME' term_lower = term.lower() diff --git a/tools/skillgen/expected/graphify__skills__opencode__references__transcribe.md b/tools/skillgen/expected/graphify__skills__opencode__references__transcribe.md index bbe8b93e45..b967f83799 100644 --- a/tools/skillgen/expected/graphify__skills__opencode__references__transcribe.md +++ b/tools/skillgen/expected/graphify__skills__opencode__references__transcribe.md @@ -8,11 +8,13 @@ Skip this step entirely if `detect` returned zero `video` files. Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. -**Strategy:** If `graphify-out/.graphify_analysis.json` exists from a previous run, read its top god-node labels and write a one-sentence domain hint from them. Otherwise use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` Pass the chosen prompt to Whisper as its initial prompt. No separate API call needed. +**Strategy:** Read the god nodes from `graphify-out/.graphify_detect.json` (or the analysis file if it exists from a previous run). You are already a language model — write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. + +**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` **Step 1 - Write the Whisper prompt yourself.** -Read the top god-node labels from previous analysis when available, then compose a short domain hint sentence. With no previous analysis, use the generic fallback above. +Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: - Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` - Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` diff --git a/tools/skillgen/expected/graphify__skills__opencode__references__update.md b/tools/skillgen/expected/graphify__skills__opencode__references__update.md index 058ca8e6d2..3632fd4126 100644 --- a/tools/skillgen/expected/graphify__skills__opencode__references__update.md +++ b/tools/skillgen/expected/graphify__skills__opencode__references__update.md @@ -12,7 +12,7 @@ import sys, json from graphify.detect import detect_incremental, save_manifest from pathlib import Path -result = detect_incremental(Path(sys.argv[1])) +result = detect_incremental(Path('INPUT_PATH')) new_total = result.get('new_total', 0) print(json.dumps(result, indent=2, ensure_ascii=False)) Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") @@ -24,7 +24,7 @@ if deleted: print(f'{len(deleted)} deleted file(s) to prune.') if new_total > 0: print(f'{new_total} new/changed file(s) to re-extract.') -" "INPUT_PATH" +" ``` Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: @@ -84,7 +84,7 @@ Then: ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from pathlib import Path from graphify.build import build_merge from graphify.detect import save_manifest @@ -113,7 +113,7 @@ G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, - root=sys.argv[1], + root='INPUT_PATH', directed=IS_DIRECTED, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -150,7 +150,7 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) # is lost forever (#2015). Mirrors the library extract path # (cli._stamped_manifest_files + clear_semantic + scan_corpus). from graphify.cli import _stamped_manifest_files -_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path(sys.argv[1])) +_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) # Changed semantic files dispatched this run but NOT stamped had their chunk fail # or be omitted; clear any stale semantic_hash so they are re-queued (#1948). _sem_types = ('document', 'paper', 'image') @@ -160,9 +160,9 @@ _cleared = _dispatched - _stamped # scan_corpus = the RAW full corpus so in-root files newly excluded since last run # are dropped rather than masquerading as deletions; untouched rows preserved (#1908). _scan = {f for fl in incremental['files'].values() for f in fl} -save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) print('[graphify update] Manifest saved.') -" "INPUT_PATH" +" ``` Then run Steps 4–8 on the merged graph as normal. @@ -174,7 +174,7 @@ $(cat graphify-out/.graphify_python) -c " import json from graphify.analyze import graph_diff from graphify.build import build_from_json -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph import networkx as nx from pathlib import Path @@ -184,7 +184,7 @@ new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(e G_new = build_from_json(new_extract, directed=IS_DIRECTED) if old_data: - G_old = load_node_link_graph(old_data) + G_old = json_graph.node_link_graph(old_data, edges='links') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: diff --git a/tools/skillgen/expected/graphify__skills__pi__references__add-watch.md b/tools/skillgen/expected/graphify__skills__pi__references__add-watch.md index fcbe0cd488..77844343e1 100644 --- a/tools/skillgen/expected/graphify__skills__pi__references__add-watch.md +++ b/tools/skillgen/expected/graphify__skills__pi__references__add-watch.md @@ -13,7 +13,7 @@ from graphify.ingest import ingest from pathlib import Path try: - out = ingest(sys.argv[1], Path('./raw'), author=sys.argv[2] or None, contributor=sys.argv[3] or None) + out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') print(f'Saved to {out}') except ValueError as e: print(f'error: {e}', file=sys.stderr) @@ -21,10 +21,10 @@ except ValueError as e: except RuntimeError as e: print(f'error: {e}', file=sys.stderr) sys.exit(1) -" "URL" "AUTHOR" "CONTRIBUTOR" +" ``` -Replace `URL` with the actual URL and pass the user's `AUTHOR` or `CONTRIBUTOR` when provided; use an empty string for either omitted value. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on the scan root recorded in `graphify-out/.graphify_root` to merge the new file into the existing graph. +Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. Supported URL types (auto-detected): - YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`) @@ -41,7 +41,7 @@ Supported URL types (auto-detected): Start a background watcher that monitors a folder and auto-updates the graph when files change. ```bash -$(cat graphify-out/.graphify_python) -m graphify.watch "INPUT_PATH" --debounce 3 +$(cat graphify-out/.graphify_python) -m graphify.watch INPUT_PATH --debounce 3 ``` Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: diff --git a/tools/skillgen/expected/graphify__skills__pi__references__exports.md b/tools/skillgen/expected/graphify__skills__pi__references__exports.md index 89ce996fb7..242ff868e0 100644 --- a/tools/skillgen/expected/graphify__skills__pi__references__exports.md +++ b/tools/skillgen/expected/graphify__skills__pi__references__exports.md @@ -20,27 +20,13 @@ graphify export wiki graphify export neo4j ``` -**If `--neo4j-push `** - push directly to a running Neo4j instance. Do not run these blocks through an agent tool: its shell is non-interactive and cannot read a hidden password. Show the block for the user's shell, ask them to run it in their own terminal, and wait for confirmation before continuing. Never ask them to paste the password into chat. - -POSIX shell: +**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: ```bash -NEO4J_PASSWORD="$("$(cat graphify-out/.graphify_python)" -c 'import getpass; print(getpass.getpass("Neo4j password: "))')" -export NEO4J_PASSWORD -graphify export neo4j --push bolt://localhost:7687 --user neo4j -unset NEO4J_PASSWORD -``` - -PowerShell: - -```powershell -$credential = Read-Host "Neo4j password" -AsSecureString -$env:NEO4J_PASSWORD = [System.Net.NetworkCredential]::new("", $credential).Password -graphify export neo4j --push bolt://localhost:7687 --user neo4j -Remove-Item Env:NEO4J_PASSWORD +graphify export neo4j --push bolt://localhost:7687 --user neo4j --password PASSWORD ``` -Default URI is `bolt://localhost:7687`, default user is `neo4j`. The command reads the password from `NEO4J_PASSWORD`. Uses MERGE - safe to re-run without creating duplicates. +Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. ### Step 7a - FalkorDB export (only if --falkordb or --falkordb-push flag) diff --git a/tools/skillgen/expected/graphify__skills__pi__references__extraction-spec.md b/tools/skillgen/expected/graphify__skills__pi__references__extraction-spec.md index 8aaa23729b..4b278b28d3 100644 --- a/tools/skillgen/expected/graphify__skills__pi__references__extraction-spec.md +++ b/tools/skillgen/expected/graphify__skills__pi__references__extraction-spec.md @@ -25,7 +25,7 @@ Rules: Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format `{stem}_{entity}` where stem is the full repo-relative path with the extension dropped, every segment joined with `_` (each lowercased with non-alphanumeric chars replaced by `_`) and entity is the symbol name similarly normalized. Use every directory level, not just the immediate parent. `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`. Top-level files use just the filename stem. This must match the AST extractor's ID. Never append chunk or sequence suffixes — IDs must be deterministic from the label alone. Output exactly this JSON (no other text): -{"nodes":[{"id":"src_auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} +{"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} source_file RULE: set source_file to the FILE_LIST path for that file VERBATIM (absolute, no shortening to basename, no re-relativizing, no separator change). Keeps full build and --update on one base so build_merge's replace matches instead of duplicating. ``` diff --git a/tools/skillgen/expected/graphify__skills__pi__references__query.md b/tools/skillgen/expected/graphify__skills__pi__references__query.md index 6dd627863c..56565eb782 100644 --- a/tools/skillgen/expected/graphify__skills__pi__references__query.md +++ b/tools/skillgen/expected/graphify__skills__pi__references__query.md @@ -79,12 +79,12 @@ If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal ```bash $(cat graphify-out/.graphify_python) -c " import sys, json -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph import networkx as nx from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' @@ -197,11 +197,11 @@ If the CLI is unavailable, run it inline: $(cat graphify-out/.graphify_python) -c " import json, sys import networkx as nx -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') a_term = 'NODE_A' b_term = 'NODE_B' @@ -265,11 +265,11 @@ If the CLI is unavailable, run it inline: $(cat graphify-out/.graphify_python) -c " import json, sys import networkx as nx -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') term = 'NODE_NAME' term_lower = term.lower() diff --git a/tools/skillgen/expected/graphify__skills__pi__references__transcribe.md b/tools/skillgen/expected/graphify__skills__pi__references__transcribe.md index bbe8b93e45..b967f83799 100644 --- a/tools/skillgen/expected/graphify__skills__pi__references__transcribe.md +++ b/tools/skillgen/expected/graphify__skills__pi__references__transcribe.md @@ -8,11 +8,13 @@ Skip this step entirely if `detect` returned zero `video` files. Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. -**Strategy:** If `graphify-out/.graphify_analysis.json` exists from a previous run, read its top god-node labels and write a one-sentence domain hint from them. Otherwise use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` Pass the chosen prompt to Whisper as its initial prompt. No separate API call needed. +**Strategy:** Read the god nodes from `graphify-out/.graphify_detect.json` (or the analysis file if it exists from a previous run). You are already a language model — write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. + +**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` **Step 1 - Write the Whisper prompt yourself.** -Read the top god-node labels from previous analysis when available, then compose a short domain hint sentence. With no previous analysis, use the generic fallback above. +Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: - Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` - Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` diff --git a/tools/skillgen/expected/graphify__skills__pi__references__update.md b/tools/skillgen/expected/graphify__skills__pi__references__update.md index 058ca8e6d2..3632fd4126 100644 --- a/tools/skillgen/expected/graphify__skills__pi__references__update.md +++ b/tools/skillgen/expected/graphify__skills__pi__references__update.md @@ -12,7 +12,7 @@ import sys, json from graphify.detect import detect_incremental, save_manifest from pathlib import Path -result = detect_incremental(Path(sys.argv[1])) +result = detect_incremental(Path('INPUT_PATH')) new_total = result.get('new_total', 0) print(json.dumps(result, indent=2, ensure_ascii=False)) Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") @@ -24,7 +24,7 @@ if deleted: print(f'{len(deleted)} deleted file(s) to prune.') if new_total > 0: print(f'{new_total} new/changed file(s) to re-extract.') -" "INPUT_PATH" +" ``` Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: @@ -84,7 +84,7 @@ Then: ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from pathlib import Path from graphify.build import build_merge from graphify.detect import save_manifest @@ -113,7 +113,7 @@ G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, - root=sys.argv[1], + root='INPUT_PATH', directed=IS_DIRECTED, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -150,7 +150,7 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) # is lost forever (#2015). Mirrors the library extract path # (cli._stamped_manifest_files + clear_semantic + scan_corpus). from graphify.cli import _stamped_manifest_files -_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path(sys.argv[1])) +_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) # Changed semantic files dispatched this run but NOT stamped had their chunk fail # or be omitted; clear any stale semantic_hash so they are re-queued (#1948). _sem_types = ('document', 'paper', 'image') @@ -160,9 +160,9 @@ _cleared = _dispatched - _stamped # scan_corpus = the RAW full corpus so in-root files newly excluded since last run # are dropped rather than masquerading as deletions; untouched rows preserved (#1908). _scan = {f for fl in incremental['files'].values() for f in fl} -save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) print('[graphify update] Manifest saved.') -" "INPUT_PATH" +" ``` Then run Steps 4–8 on the merged graph as normal. @@ -174,7 +174,7 @@ $(cat graphify-out/.graphify_python) -c " import json from graphify.analyze import graph_diff from graphify.build import build_from_json -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph import networkx as nx from pathlib import Path @@ -184,7 +184,7 @@ new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(e G_new = build_from_json(new_extract, directed=IS_DIRECTED) if old_data: - G_old = load_node_link_graph(old_data) + G_old = json_graph.node_link_graph(old_data, edges='links') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: diff --git a/tools/skillgen/expected/graphify__skills__trae__references__add-watch.md b/tools/skillgen/expected/graphify__skills__trae__references__add-watch.md index fcbe0cd488..77844343e1 100644 --- a/tools/skillgen/expected/graphify__skills__trae__references__add-watch.md +++ b/tools/skillgen/expected/graphify__skills__trae__references__add-watch.md @@ -13,7 +13,7 @@ from graphify.ingest import ingest from pathlib import Path try: - out = ingest(sys.argv[1], Path('./raw'), author=sys.argv[2] or None, contributor=sys.argv[3] or None) + out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') print(f'Saved to {out}') except ValueError as e: print(f'error: {e}', file=sys.stderr) @@ -21,10 +21,10 @@ except ValueError as e: except RuntimeError as e: print(f'error: {e}', file=sys.stderr) sys.exit(1) -" "URL" "AUTHOR" "CONTRIBUTOR" +" ``` -Replace `URL` with the actual URL and pass the user's `AUTHOR` or `CONTRIBUTOR` when provided; use an empty string for either omitted value. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on the scan root recorded in `graphify-out/.graphify_root` to merge the new file into the existing graph. +Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. Supported URL types (auto-detected): - YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`) @@ -41,7 +41,7 @@ Supported URL types (auto-detected): Start a background watcher that monitors a folder and auto-updates the graph when files change. ```bash -$(cat graphify-out/.graphify_python) -m graphify.watch "INPUT_PATH" --debounce 3 +$(cat graphify-out/.graphify_python) -m graphify.watch INPUT_PATH --debounce 3 ``` Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: diff --git a/tools/skillgen/expected/graphify__skills__trae__references__exports.md b/tools/skillgen/expected/graphify__skills__trae__references__exports.md index 89ce996fb7..242ff868e0 100644 --- a/tools/skillgen/expected/graphify__skills__trae__references__exports.md +++ b/tools/skillgen/expected/graphify__skills__trae__references__exports.md @@ -20,27 +20,13 @@ graphify export wiki graphify export neo4j ``` -**If `--neo4j-push `** - push directly to a running Neo4j instance. Do not run these blocks through an agent tool: its shell is non-interactive and cannot read a hidden password. Show the block for the user's shell, ask them to run it in their own terminal, and wait for confirmation before continuing. Never ask them to paste the password into chat. - -POSIX shell: +**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: ```bash -NEO4J_PASSWORD="$("$(cat graphify-out/.graphify_python)" -c 'import getpass; print(getpass.getpass("Neo4j password: "))')" -export NEO4J_PASSWORD -graphify export neo4j --push bolt://localhost:7687 --user neo4j -unset NEO4J_PASSWORD -``` - -PowerShell: - -```powershell -$credential = Read-Host "Neo4j password" -AsSecureString -$env:NEO4J_PASSWORD = [System.Net.NetworkCredential]::new("", $credential).Password -graphify export neo4j --push bolt://localhost:7687 --user neo4j -Remove-Item Env:NEO4J_PASSWORD +graphify export neo4j --push bolt://localhost:7687 --user neo4j --password PASSWORD ``` -Default URI is `bolt://localhost:7687`, default user is `neo4j`. The command reads the password from `NEO4J_PASSWORD`. Uses MERGE - safe to re-run without creating duplicates. +Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. ### Step 7a - FalkorDB export (only if --falkordb or --falkordb-push flag) diff --git a/tools/skillgen/expected/graphify__skills__trae__references__extraction-spec.md b/tools/skillgen/expected/graphify__skills__trae__references__extraction-spec.md index a5a12a575e..388df7674f 100644 --- a/tools/skillgen/expected/graphify__skills__trae__references__extraction-spec.md +++ b/tools/skillgen/expected/graphify__skills__trae__references__extraction-spec.md @@ -61,7 +61,7 @@ confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a d Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is the **full repo-relative path with the extension dropped**, every path segment kept and joined with `_` (each segment lowercased with non-alphanumeric chars replaced by `_`), and entity is the symbol name similarly normalized. Use every directory level, not just the immediate parent — this keeps same-named files in different directories distinct. Examples: `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `lib_utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`; `docs/v1/api/README.md` + `getUser` → `docs_v1_api_readme_getuser`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or only the immediate parent (e.g., `auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project built under the old immediate-parent format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. Generate the extraction JSON matching this schema exactly: -{"nodes":[{"id":"src_auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} +{"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. diff --git a/tools/skillgen/expected/graphify__skills__trae__references__query.md b/tools/skillgen/expected/graphify__skills__trae__references__query.md index 6dd627863c..56565eb782 100644 --- a/tools/skillgen/expected/graphify__skills__trae__references__query.md +++ b/tools/skillgen/expected/graphify__skills__trae__references__query.md @@ -79,12 +79,12 @@ If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal ```bash $(cat graphify-out/.graphify_python) -c " import sys, json -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph import networkx as nx from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' @@ -197,11 +197,11 @@ If the CLI is unavailable, run it inline: $(cat graphify-out/.graphify_python) -c " import json, sys import networkx as nx -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') a_term = 'NODE_A' b_term = 'NODE_B' @@ -265,11 +265,11 @@ If the CLI is unavailable, run it inline: $(cat graphify-out/.graphify_python) -c " import json, sys import networkx as nx -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') term = 'NODE_NAME' term_lower = term.lower() diff --git a/tools/skillgen/expected/graphify__skills__trae__references__transcribe.md b/tools/skillgen/expected/graphify__skills__trae__references__transcribe.md index bbe8b93e45..b967f83799 100644 --- a/tools/skillgen/expected/graphify__skills__trae__references__transcribe.md +++ b/tools/skillgen/expected/graphify__skills__trae__references__transcribe.md @@ -8,11 +8,13 @@ Skip this step entirely if `detect` returned zero `video` files. Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. -**Strategy:** If `graphify-out/.graphify_analysis.json` exists from a previous run, read its top god-node labels and write a one-sentence domain hint from them. Otherwise use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` Pass the chosen prompt to Whisper as its initial prompt. No separate API call needed. +**Strategy:** Read the god nodes from `graphify-out/.graphify_detect.json` (or the analysis file if it exists from a previous run). You are already a language model — write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. + +**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` **Step 1 - Write the Whisper prompt yourself.** -Read the top god-node labels from previous analysis when available, then compose a short domain hint sentence. With no previous analysis, use the generic fallback above. +Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: - Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` - Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` diff --git a/tools/skillgen/expected/graphify__skills__trae__references__update.md b/tools/skillgen/expected/graphify__skills__trae__references__update.md index 058ca8e6d2..3632fd4126 100644 --- a/tools/skillgen/expected/graphify__skills__trae__references__update.md +++ b/tools/skillgen/expected/graphify__skills__trae__references__update.md @@ -12,7 +12,7 @@ import sys, json from graphify.detect import detect_incremental, save_manifest from pathlib import Path -result = detect_incremental(Path(sys.argv[1])) +result = detect_incremental(Path('INPUT_PATH')) new_total = result.get('new_total', 0) print(json.dumps(result, indent=2, ensure_ascii=False)) Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") @@ -24,7 +24,7 @@ if deleted: print(f'{len(deleted)} deleted file(s) to prune.') if new_total > 0: print(f'{new_total} new/changed file(s) to re-extract.') -" "INPUT_PATH" +" ``` Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: @@ -84,7 +84,7 @@ Then: ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from pathlib import Path from graphify.build import build_merge from graphify.detect import save_manifest @@ -113,7 +113,7 @@ G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, - root=sys.argv[1], + root='INPUT_PATH', directed=IS_DIRECTED, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -150,7 +150,7 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) # is lost forever (#2015). Mirrors the library extract path # (cli._stamped_manifest_files + clear_semantic + scan_corpus). from graphify.cli import _stamped_manifest_files -_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path(sys.argv[1])) +_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) # Changed semantic files dispatched this run but NOT stamped had their chunk fail # or be omitted; clear any stale semantic_hash so they are re-queued (#1948). _sem_types = ('document', 'paper', 'image') @@ -160,9 +160,9 @@ _cleared = _dispatched - _stamped # scan_corpus = the RAW full corpus so in-root files newly excluded since last run # are dropped rather than masquerading as deletions; untouched rows preserved (#1908). _scan = {f for fl in incremental['files'].values() for f in fl} -save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) print('[graphify update] Manifest saved.') -" "INPUT_PATH" +" ``` Then run Steps 4–8 on the merged graph as normal. @@ -174,7 +174,7 @@ $(cat graphify-out/.graphify_python) -c " import json from graphify.analyze import graph_diff from graphify.build import build_from_json -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph import networkx as nx from pathlib import Path @@ -184,7 +184,7 @@ new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(e G_new = build_from_json(new_extract, directed=IS_DIRECTED) if old_data: - G_old = load_node_link_graph(old_data) + G_old = json_graph.node_link_graph(old_data, edges='links') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: diff --git a/tools/skillgen/expected/graphify__skills__vscode__references__add-watch.md b/tools/skillgen/expected/graphify__skills__vscode__references__add-watch.md index fcbe0cd488..77844343e1 100644 --- a/tools/skillgen/expected/graphify__skills__vscode__references__add-watch.md +++ b/tools/skillgen/expected/graphify__skills__vscode__references__add-watch.md @@ -13,7 +13,7 @@ from graphify.ingest import ingest from pathlib import Path try: - out = ingest(sys.argv[1], Path('./raw'), author=sys.argv[2] or None, contributor=sys.argv[3] or None) + out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') print(f'Saved to {out}') except ValueError as e: print(f'error: {e}', file=sys.stderr) @@ -21,10 +21,10 @@ except ValueError as e: except RuntimeError as e: print(f'error: {e}', file=sys.stderr) sys.exit(1) -" "URL" "AUTHOR" "CONTRIBUTOR" +" ``` -Replace `URL` with the actual URL and pass the user's `AUTHOR` or `CONTRIBUTOR` when provided; use an empty string for either omitted value. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on the scan root recorded in `graphify-out/.graphify_root` to merge the new file into the existing graph. +Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. Supported URL types (auto-detected): - YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`) @@ -41,7 +41,7 @@ Supported URL types (auto-detected): Start a background watcher that monitors a folder and auto-updates the graph when files change. ```bash -$(cat graphify-out/.graphify_python) -m graphify.watch "INPUT_PATH" --debounce 3 +$(cat graphify-out/.graphify_python) -m graphify.watch INPUT_PATH --debounce 3 ``` Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: diff --git a/tools/skillgen/expected/graphify__skills__vscode__references__exports.md b/tools/skillgen/expected/graphify__skills__vscode__references__exports.md index 89ce996fb7..242ff868e0 100644 --- a/tools/skillgen/expected/graphify__skills__vscode__references__exports.md +++ b/tools/skillgen/expected/graphify__skills__vscode__references__exports.md @@ -20,27 +20,13 @@ graphify export wiki graphify export neo4j ``` -**If `--neo4j-push `** - push directly to a running Neo4j instance. Do not run these blocks through an agent tool: its shell is non-interactive and cannot read a hidden password. Show the block for the user's shell, ask them to run it in their own terminal, and wait for confirmation before continuing. Never ask them to paste the password into chat. - -POSIX shell: +**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: ```bash -NEO4J_PASSWORD="$("$(cat graphify-out/.graphify_python)" -c 'import getpass; print(getpass.getpass("Neo4j password: "))')" -export NEO4J_PASSWORD -graphify export neo4j --push bolt://localhost:7687 --user neo4j -unset NEO4J_PASSWORD -``` - -PowerShell: - -```powershell -$credential = Read-Host "Neo4j password" -AsSecureString -$env:NEO4J_PASSWORD = [System.Net.NetworkCredential]::new("", $credential).Password -graphify export neo4j --push bolt://localhost:7687 --user neo4j -Remove-Item Env:NEO4J_PASSWORD +graphify export neo4j --push bolt://localhost:7687 --user neo4j --password PASSWORD ``` -Default URI is `bolt://localhost:7687`, default user is `neo4j`. The command reads the password from `NEO4J_PASSWORD`. Uses MERGE - safe to re-run without creating duplicates. +Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. ### Step 7a - FalkorDB export (only if --falkordb or --falkordb-push flag) diff --git a/tools/skillgen/expected/graphify__skills__vscode__references__extraction-spec.md b/tools/skillgen/expected/graphify__skills__vscode__references__extraction-spec.md index a5a12a575e..388df7674f 100644 --- a/tools/skillgen/expected/graphify__skills__vscode__references__extraction-spec.md +++ b/tools/skillgen/expected/graphify__skills__vscode__references__extraction-spec.md @@ -61,7 +61,7 @@ confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a d Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is the **full repo-relative path with the extension dropped**, every path segment kept and joined with `_` (each segment lowercased with non-alphanumeric chars replaced by `_`), and entity is the symbol name similarly normalized. Use every directory level, not just the immediate parent — this keeps same-named files in different directories distinct. Examples: `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `lib_utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`; `docs/v1/api/README.md` + `getUser` → `docs_v1_api_readme_getuser`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or only the immediate parent (e.g., `auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project built under the old immediate-parent format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. Generate the extraction JSON matching this schema exactly: -{"nodes":[{"id":"src_auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} +{"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. diff --git a/tools/skillgen/expected/graphify__skills__vscode__references__query.md b/tools/skillgen/expected/graphify__skills__vscode__references__query.md index 6dd627863c..56565eb782 100644 --- a/tools/skillgen/expected/graphify__skills__vscode__references__query.md +++ b/tools/skillgen/expected/graphify__skills__vscode__references__query.md @@ -79,12 +79,12 @@ If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal ```bash $(cat graphify-out/.graphify_python) -c " import sys, json -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph import networkx as nx from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' @@ -197,11 +197,11 @@ If the CLI is unavailable, run it inline: $(cat graphify-out/.graphify_python) -c " import json, sys import networkx as nx -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') a_term = 'NODE_A' b_term = 'NODE_B' @@ -265,11 +265,11 @@ If the CLI is unavailable, run it inline: $(cat graphify-out/.graphify_python) -c " import json, sys import networkx as nx -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') term = 'NODE_NAME' term_lower = term.lower() diff --git a/tools/skillgen/expected/graphify__skills__vscode__references__transcribe.md b/tools/skillgen/expected/graphify__skills__vscode__references__transcribe.md index bbe8b93e45..b967f83799 100644 --- a/tools/skillgen/expected/graphify__skills__vscode__references__transcribe.md +++ b/tools/skillgen/expected/graphify__skills__vscode__references__transcribe.md @@ -8,11 +8,13 @@ Skip this step entirely if `detect` returned zero `video` files. Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. -**Strategy:** If `graphify-out/.graphify_analysis.json` exists from a previous run, read its top god-node labels and write a one-sentence domain hint from them. Otherwise use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` Pass the chosen prompt to Whisper as its initial prompt. No separate API call needed. +**Strategy:** Read the god nodes from `graphify-out/.graphify_detect.json` (or the analysis file if it exists from a previous run). You are already a language model — write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. + +**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` **Step 1 - Write the Whisper prompt yourself.** -Read the top god-node labels from previous analysis when available, then compose a short domain hint sentence. With no previous analysis, use the generic fallback above. +Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: - Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` - Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` diff --git a/tools/skillgen/expected/graphify__skills__vscode__references__update.md b/tools/skillgen/expected/graphify__skills__vscode__references__update.md index 058ca8e6d2..3632fd4126 100644 --- a/tools/skillgen/expected/graphify__skills__vscode__references__update.md +++ b/tools/skillgen/expected/graphify__skills__vscode__references__update.md @@ -12,7 +12,7 @@ import sys, json from graphify.detect import detect_incremental, save_manifest from pathlib import Path -result = detect_incremental(Path(sys.argv[1])) +result = detect_incremental(Path('INPUT_PATH')) new_total = result.get('new_total', 0) print(json.dumps(result, indent=2, ensure_ascii=False)) Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") @@ -24,7 +24,7 @@ if deleted: print(f'{len(deleted)} deleted file(s) to prune.') if new_total > 0: print(f'{new_total} new/changed file(s) to re-extract.') -" "INPUT_PATH" +" ``` Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: @@ -84,7 +84,7 @@ Then: ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from pathlib import Path from graphify.build import build_merge from graphify.detect import save_manifest @@ -113,7 +113,7 @@ G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, - root=sys.argv[1], + root='INPUT_PATH', directed=IS_DIRECTED, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -150,7 +150,7 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) # is lost forever (#2015). Mirrors the library extract path # (cli._stamped_manifest_files + clear_semantic + scan_corpus). from graphify.cli import _stamped_manifest_files -_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path(sys.argv[1])) +_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) # Changed semantic files dispatched this run but NOT stamped had their chunk fail # or be omitted; clear any stale semantic_hash so they are re-queued (#1948). _sem_types = ('document', 'paper', 'image') @@ -160,9 +160,9 @@ _cleared = _dispatched - _stamped # scan_corpus = the RAW full corpus so in-root files newly excluded since last run # are dropped rather than masquerading as deletions; untouched rows preserved (#1908). _scan = {f for fl in incremental['files'].values() for f in fl} -save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) print('[graphify update] Manifest saved.') -" "INPUT_PATH" +" ``` Then run Steps 4–8 on the merged graph as normal. @@ -174,7 +174,7 @@ $(cat graphify-out/.graphify_python) -c " import json from graphify.analyze import graph_diff from graphify.build import build_from_json -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph import networkx as nx from pathlib import Path @@ -184,7 +184,7 @@ new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(e G_new = build_from_json(new_extract, directed=IS_DIRECTED) if old_data: - G_old = load_node_link_graph(old_data) + G_old = json_graph.node_link_graph(old_data, edges='links') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: diff --git a/tools/skillgen/expected/graphify__skills__windows__references__add-watch.md b/tools/skillgen/expected/graphify__skills__windows__references__add-watch.md index fcbe0cd488..77844343e1 100644 --- a/tools/skillgen/expected/graphify__skills__windows__references__add-watch.md +++ b/tools/skillgen/expected/graphify__skills__windows__references__add-watch.md @@ -13,7 +13,7 @@ from graphify.ingest import ingest from pathlib import Path try: - out = ingest(sys.argv[1], Path('./raw'), author=sys.argv[2] or None, contributor=sys.argv[3] or None) + out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') print(f'Saved to {out}') except ValueError as e: print(f'error: {e}', file=sys.stderr) @@ -21,10 +21,10 @@ except ValueError as e: except RuntimeError as e: print(f'error: {e}', file=sys.stderr) sys.exit(1) -" "URL" "AUTHOR" "CONTRIBUTOR" +" ``` -Replace `URL` with the actual URL and pass the user's `AUTHOR` or `CONTRIBUTOR` when provided; use an empty string for either omitted value. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on the scan root recorded in `graphify-out/.graphify_root` to merge the new file into the existing graph. +Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. Supported URL types (auto-detected): - YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`) @@ -41,7 +41,7 @@ Supported URL types (auto-detected): Start a background watcher that monitors a folder and auto-updates the graph when files change. ```bash -$(cat graphify-out/.graphify_python) -m graphify.watch "INPUT_PATH" --debounce 3 +$(cat graphify-out/.graphify_python) -m graphify.watch INPUT_PATH --debounce 3 ``` Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: diff --git a/tools/skillgen/expected/graphify__skills__windows__references__exports.md b/tools/skillgen/expected/graphify__skills__windows__references__exports.md index 89ce996fb7..242ff868e0 100644 --- a/tools/skillgen/expected/graphify__skills__windows__references__exports.md +++ b/tools/skillgen/expected/graphify__skills__windows__references__exports.md @@ -20,27 +20,13 @@ graphify export wiki graphify export neo4j ``` -**If `--neo4j-push `** - push directly to a running Neo4j instance. Do not run these blocks through an agent tool: its shell is non-interactive and cannot read a hidden password. Show the block for the user's shell, ask them to run it in their own terminal, and wait for confirmation before continuing. Never ask them to paste the password into chat. - -POSIX shell: +**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: ```bash -NEO4J_PASSWORD="$("$(cat graphify-out/.graphify_python)" -c 'import getpass; print(getpass.getpass("Neo4j password: "))')" -export NEO4J_PASSWORD -graphify export neo4j --push bolt://localhost:7687 --user neo4j -unset NEO4J_PASSWORD -``` - -PowerShell: - -```powershell -$credential = Read-Host "Neo4j password" -AsSecureString -$env:NEO4J_PASSWORD = [System.Net.NetworkCredential]::new("", $credential).Password -graphify export neo4j --push bolt://localhost:7687 --user neo4j -Remove-Item Env:NEO4J_PASSWORD +graphify export neo4j --push bolt://localhost:7687 --user neo4j --password PASSWORD ``` -Default URI is `bolt://localhost:7687`, default user is `neo4j`. The command reads the password from `NEO4J_PASSWORD`. Uses MERGE - safe to re-run without creating duplicates. +Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. ### Step 7a - FalkorDB export (only if --falkordb or --falkordb-push flag) diff --git a/tools/skillgen/expected/graphify__skills__windows__references__extraction-spec.md b/tools/skillgen/expected/graphify__skills__windows__references__extraction-spec.md index a5a12a575e..388df7674f 100644 --- a/tools/skillgen/expected/graphify__skills__windows__references__extraction-spec.md +++ b/tools/skillgen/expected/graphify__skills__windows__references__extraction-spec.md @@ -61,7 +61,7 @@ confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a d Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is the **full repo-relative path with the extension dropped**, every path segment kept and joined with `_` (each segment lowercased with non-alphanumeric chars replaced by `_`), and entity is the symbol name similarly normalized. Use every directory level, not just the immediate parent — this keeps same-named files in different directories distinct. Examples: `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `lib_utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`; `docs/v1/api/README.md` + `getUser` → `docs_v1_api_readme_getuser`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or only the immediate parent (e.g., `auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project built under the old immediate-parent format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. Generate the extraction JSON matching this schema exactly: -{"nodes":[{"id":"src_auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} +{"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. diff --git a/tools/skillgen/expected/graphify__skills__windows__references__query.md b/tools/skillgen/expected/graphify__skills__windows__references__query.md index 6dd627863c..56565eb782 100644 --- a/tools/skillgen/expected/graphify__skills__windows__references__query.md +++ b/tools/skillgen/expected/graphify__skills__windows__references__query.md @@ -79,12 +79,12 @@ If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal ```bash $(cat graphify-out/.graphify_python) -c " import sys, json -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph import networkx as nx from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' @@ -197,11 +197,11 @@ If the CLI is unavailable, run it inline: $(cat graphify-out/.graphify_python) -c " import json, sys import networkx as nx -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') a_term = 'NODE_A' b_term = 'NODE_B' @@ -265,11 +265,11 @@ If the CLI is unavailable, run it inline: $(cat graphify-out/.graphify_python) -c " import json, sys import networkx as nx -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') term = 'NODE_NAME' term_lower = term.lower() diff --git a/tools/skillgen/expected/graphify__skills__windows__references__transcribe.md b/tools/skillgen/expected/graphify__skills__windows__references__transcribe.md index bbe8b93e45..b967f83799 100644 --- a/tools/skillgen/expected/graphify__skills__windows__references__transcribe.md +++ b/tools/skillgen/expected/graphify__skills__windows__references__transcribe.md @@ -8,11 +8,13 @@ Skip this step entirely if `detect` returned zero `video` files. Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. -**Strategy:** If `graphify-out/.graphify_analysis.json` exists from a previous run, read its top god-node labels and write a one-sentence domain hint from them. Otherwise use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` Pass the chosen prompt to Whisper as its initial prompt. No separate API call needed. +**Strategy:** Read the god nodes from `graphify-out/.graphify_detect.json` (or the analysis file if it exists from a previous run). You are already a language model — write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. + +**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` **Step 1 - Write the Whisper prompt yourself.** -Read the top god-node labels from previous analysis when available, then compose a short domain hint sentence. With no previous analysis, use the generic fallback above. +Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: - Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` - Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` diff --git a/tools/skillgen/expected/graphify__skills__windows__references__update.md b/tools/skillgen/expected/graphify__skills__windows__references__update.md index 058ca8e6d2..3632fd4126 100644 --- a/tools/skillgen/expected/graphify__skills__windows__references__update.md +++ b/tools/skillgen/expected/graphify__skills__windows__references__update.md @@ -12,7 +12,7 @@ import sys, json from graphify.detect import detect_incremental, save_manifest from pathlib import Path -result = detect_incremental(Path(sys.argv[1])) +result = detect_incremental(Path('INPUT_PATH')) new_total = result.get('new_total', 0) print(json.dumps(result, indent=2, ensure_ascii=False)) Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") @@ -24,7 +24,7 @@ if deleted: print(f'{len(deleted)} deleted file(s) to prune.') if new_total > 0: print(f'{new_total} new/changed file(s) to re-extract.') -" "INPUT_PATH" +" ``` Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: @@ -84,7 +84,7 @@ Then: ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from pathlib import Path from graphify.build import build_merge from graphify.detect import save_manifest @@ -113,7 +113,7 @@ G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, - root=sys.argv[1], + root='INPUT_PATH', directed=IS_DIRECTED, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -150,7 +150,7 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) # is lost forever (#2015). Mirrors the library extract path # (cli._stamped_manifest_files + clear_semantic + scan_corpus). from graphify.cli import _stamped_manifest_files -_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path(sys.argv[1])) +_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) # Changed semantic files dispatched this run but NOT stamped had their chunk fail # or be omitted; clear any stale semantic_hash so they are re-queued (#1948). _sem_types = ('document', 'paper', 'image') @@ -160,9 +160,9 @@ _cleared = _dispatched - _stamped # scan_corpus = the RAW full corpus so in-root files newly excluded since last run # are dropped rather than masquerading as deletions; untouched rows preserved (#1908). _scan = {f for fl in incremental['files'].values() for f in fl} -save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) print('[graphify update] Manifest saved.') -" "INPUT_PATH" +" ``` Then run Steps 4–8 on the merged graph as normal. @@ -174,7 +174,7 @@ $(cat graphify-out/.graphify_python) -c " import json from graphify.analyze import graph_diff from graphify.build import build_from_json -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph import networkx as nx from pathlib import Path @@ -184,7 +184,7 @@ new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(e G_new = build_from_json(new_extract, directed=IS_DIRECTED) if old_data: - G_old = load_node_link_graph(old_data) + G_old = json_graph.node_link_graph(old_data, edges='links') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: diff --git a/tools/skillgen/fragments/core/aider.md b/tools/skillgen/fragments/core/aider.md index aacd9f31fc..4996beb787 100644 --- a/tools/skillgen/fragments/core/aider.md +++ b/tools/skillgen/fragments/core/aider.md @@ -101,12 +101,11 @@ If the import succeeds, print nothing and move straight to Step 2. ```bash $(cat graphify-out/.graphify_python) -c " import json -import sys from graphify.detect import detect from pathlib import Path -result = detect(Path(sys.argv[1])) +result = detect(Path('INPUT_PATH')) print(json.dumps(result)) -" "INPUT_PATH" > .graphify_detect.json +" > .graphify_detect.json ``` Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: @@ -134,11 +133,13 @@ Skip this step entirely if `detect` returned zero `video` files. Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. -**Strategy:** If an analysis file exists from a previous run, read its top god-node labels and write a one-sentence domain hint from them. Otherwise use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` Pass the chosen prompt to Whisper as its initial prompt. No separate API call needed. +**Strategy:** Read the god nodes from the detect output or analysis file. You are already a language model - write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. + +**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` **Step 1 - Write the Whisper prompt yourself.** -Read the top god-node labels from previous analysis when available, then compose a short domain hint sentence. With no previous analysis, use the generic fallback above. +Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: - Labels: `transformer, attention, encoder, decoder` -> `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` - Labels: `kubernetes, deployment, pod, helm` -> `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` @@ -425,7 +426,7 @@ wrote = to_json(G, communities, 'graphify-out/graph.json') if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (fewer nodes than the existing graph). Run a full rebuild to be safe.') raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, sys.argv[1], suggested_questions=questions) +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) Path('graphify-out/GRAPH_REPORT.md').write_text(report) analysis = { @@ -437,7 +438,7 @@ analysis = { } Path('.graphify_analysis.json').write_text(json.dumps(analysis, indent=2)) print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" "INPUT_PATH" +" ``` If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. @@ -475,18 +476,17 @@ labels = LABELS_DICT # Regenerate questions with real community labels (labels affect question phrasing) questions = suggest_questions(G, communities, labels) -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, sys.argv[1], suggested_questions=questions) +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report) +Path('.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()})) # Re-export so graph.json nodes carry the curated community_name (#2490). # Same extraction as Step 4, so the #479 shrink-guard passes on node count; # if it still refuses, surface the guard message - do not force past it. wrote = to_json(G, communities, 'graphify-out/graph.json', community_labels=labels) if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (fewer nodes than the existing graph). Run a full rebuild to be safe.') - raise SystemExit(1) -Path('graphify-out/GRAPH_REPORT.md').write_text(report) -Path('.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()})) print('Report updated with community labels') -" "INPUT_PATH" +" ``` Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). @@ -678,7 +678,6 @@ Print the output directly in chat. If `total_words <= 5000`, skip silently - the ```bash $(cat graphify-out/.graphify_python) -c " import json -import sys from pathlib import Path from datetime import datetime, timezone from graphify.detect import save_manifest @@ -689,13 +688,13 @@ extract = json.loads(Path('.graphify_extract.json').read_text()) # Stamp only semantic files that produced output so a failed chunk is re-queued next run, not lost (#2015). from graphify.cli import _stamped_manifest_files _corpus = detect.get('all_files') or detect['files'] -_manifest_files = _stamped_manifest_files(_corpus, extract, Path(sys.argv[1])) +_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) _sem_types = ('document', 'paper', 'image') _dispatched = {f for t, fl in detect['files'].items() if t in _sem_types for f in fl} _stamped = {f for fl in _manifest_files.values() for f in fl} _cleared = _dispatched - _stamped _scan = {f for fl in _corpus.values() for f in fl} -save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) # Update cumulative cost tracker input_tok = extract.get('input_tokens', 0) @@ -719,9 +718,9 @@ cost_path.write_text(json.dumps(cost, indent=2)) print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" "INPUT_PATH" +" rm -f .graphify_detect.json .graphify_extract.json .graphify_ast.json .graphify_semantic.json .graphify_analysis.json .graphify_labels.json; find . -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null -rm -f graphify-out/needs_update 2>/dev/null || true +rm -f graphify-out/.needs_update 2>/dev/null || true ``` Tell the user (omit the obsidian line unless --obsidian was given): @@ -765,7 +764,7 @@ import sys, json from graphify.detect import detect_incremental, save_manifest from pathlib import Path -result = detect_incremental(Path(sys.argv[1])) +result = detect_incremental(Path('INPUT_PATH')) new_total = result.get('new_total', 0) print(json.dumps(result, indent=2)) Path('.graphify_incremental.json').write_text(json.dumps(result)) @@ -777,7 +776,7 @@ if deleted: print(f'{len(deleted)} deleted file(s) to prune.') if new_total > 0: print(f'{new_total} new/changed file(s) to re-extract.') -" "INPUT_PATH" +" ``` If new files exist, first check whether all changed files are code files: @@ -822,13 +821,13 @@ $(cat graphify-out/.graphify_python) -c " import sys, json from graphify.build import build_from_json from graphify.export import to_json -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph import networkx as nx from pathlib import Path # Load existing graph existing_data = json.loads(Path('graphify-out/graph.json').read_text()) -G_existing = load_node_link_graph(existing_data) +G_existing = json_graph.node_link_graph(existing_data, edges='links') # Load new extraction new_extraction = json.loads(Path('.graphify_extract.json').read_text()) @@ -849,7 +848,7 @@ $(cat graphify-out/.graphify_python) -c " import json from graphify.analyze import graph_diff from graphify.build import build_from_json -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph import networkx as nx from pathlib import Path @@ -859,7 +858,7 @@ new_extract = json.loads(Path('.graphify_extract.json').read_text()) G_new = build_from_json(new_extract, directed=IS_DIRECTED) if old_data: - G_old = load_node_link_graph(old_data) + G_old = json_graph.node_link_graph(old_data, edges='links') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: @@ -885,12 +884,12 @@ from graphify.cluster import cluster, score_all from graphify.analyze import god_nodes, surprising_connections from graphify.report import generate from graphify.export import to_json -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph import networkx as nx from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text()) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') detection = {'total_files': 0, 'total_words': 99999, 'needs_graph': True, 'warning': None, 'files': {'code': [], 'document': [], 'paper': []}} @@ -954,12 +953,12 @@ Load `graphify-out/graph.json`, then: ```bash $(cat graphify-out/.graphify_python) -c " import sys, json -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph import networkx as nx from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text()) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' @@ -1069,11 +1068,11 @@ If it fails, stop and tell the user to run `/graphify ` first. $(cat graphify-out/.graphify_python) -c " import json, sys import networkx as nx -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text()) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') a_term = 'NODE_A' b_term = 'NODE_B' @@ -1142,11 +1141,11 @@ If it fails, stop and tell the user to run `/graphify ` first. $(cat graphify-out/.graphify_python) -c " import json, sys import networkx as nx -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text()) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') term = 'NODE_NAME' term_lower = term.lower() @@ -1200,7 +1199,7 @@ from graphify.ingest import ingest from pathlib import Path try: - out = ingest(sys.argv[1], Path('./raw'), author=sys.argv[2] or None, contributor=sys.argv[3] or None) + out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') print(f'Saved to {out}') except ValueError as e: print(f'error: {e}', file=sys.stderr) @@ -1208,10 +1207,10 @@ except ValueError as e: except RuntimeError as e: print(f'error: {e}', file=sys.stderr) sys.exit(1) -" "URL" "AUTHOR" "CONTRIBUTOR" +" ``` -Replace `URL` with the actual URL and pass the user's `AUTHOR` or `CONTRIBUTOR` when provided; use an empty string for either omitted value. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. +Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. Supported URL types (auto-detected): - Twitter/X → fetched via oEmbed, saved as `.md` with tweet text and author @@ -1227,7 +1226,7 @@ Supported URL types (auto-detected): Start a background watcher that monitors a folder and auto-updates the graph when files change. ```bash -python3 -m graphify.watch "INPUT_PATH" --debounce 3 +python3 -m graphify.watch INPUT_PATH --debounce 3 ``` Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: diff --git a/tools/skillgen/fragments/core/core.md b/tools/skillgen/fragments/core/core.md index bab7b4b5a5..c527a12563 100644 --- a/tools/skillgen/fragments/core/core.md +++ b/tools/skillgen/fragments/core/core.md @@ -67,15 +67,15 @@ Only when the path is one or more `https://github.com/...` URLs, or several loca ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from graphify.detect import detect from pathlib import Path -result = detect(Path(sys.argv[1])) +result = detect(Path('INPUT_PATH')) # Write the sidecar from Python, not a shell redirect, so the same block renders # on PowerShell hosts without console-encoding drift (#2528). Path('graphify-out/.graphify_detect.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") print(f'Detected {result[\"total_files\"]} files') -" "INPUT_PATH" +" ``` Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: @@ -135,6 +135,7 @@ $(cat graphify-out/.graphify_python) -c " import sys, json from graphify.extract import collect_files, extract from pathlib import Path +import json code_files = [] detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) @@ -142,13 +143,13 @@ for f in detect.get('files', {}).get('code', []): code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) if code_files: - result = extract(code_files, cache_root=Path(sys.argv[1])) + result = extract(code_files, cache_root=Path('INPUT_PATH')) Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') else: Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") print('No code files - skipping AST extraction') -" "INPUT_PATH" +" ``` #### Part B - Semantic extraction (parallel subagents) @@ -163,7 +164,7 @@ Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],' " ``` -**MANDATORY: You MUST use the subagent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the subagent tool you are doing this wrong.** +**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** Before dispatching subagents, print a timing estimate: - Load `total_words` and file counts from `graphify-out/.graphify_detect.json` @@ -179,7 +180,7 @@ SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from graphify.cache import check_semantic_cache from pathlib import Path @@ -189,7 +190,7 @@ detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encodin # every source file (#1392). Video is transcribed to a document in Step 2.5 first. all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root=sys.argv[1], prompt_file=sys.argv[2]) +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') # Always (re)write the cache file: write hits, else DELETE any leftover from a prior # run so Part C never merges a stale .graphify_cached.json (#1392). @@ -199,7 +200,7 @@ else: Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" "INPUT_PATH" "SPEC_PATH" +" ``` Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. @@ -220,7 +221,7 @@ Wait for all subagents. For each result: If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. -Merge all chunk files into `.graphify_semantic_new.json`. **After each subagent call completes, read the real token counts from the subagent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: +Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: ```bash $(cat graphify-out/.graphify_python) -c " import json, glob @@ -247,15 +248,15 @@ print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens' Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root=sys.argv[1], allowed_source_files=uncached, prompt_file=sys.argv[2]) +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') print(f'Cached {saved} files') -" "INPUT_PATH" "SPEC_PATH" +" ``` Merge cached + new results into `graphify-out/.graphify_semantic.json`: @@ -344,7 +345,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -369,7 +370,7 @@ if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') print('If this shrink is intentional (you deleted files), re-run a full build with --force.') raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, sys.argv[1], suggested_questions=questions) +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") analysis = { 'communities': {str(k): v for k, v in communities.items()}, @@ -380,7 +381,7 @@ analysis = { } Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" "INPUT_PATH" +" ``` If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. @@ -393,12 +394,12 @@ A non-destructive diagnostic on the extraction, before labeling. It surfaces edg ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from pathlib import Path from graphify.diagnostics import diagnose_extraction, format_diagnostic_report extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root=sys.argv[1]) +summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( ('dangling_endpoint_edges', 'dangling-endpoint edges'), @@ -408,7 +409,7 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0)] print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') -" "INPUT_PATH" +" ``` Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules). @@ -434,7 +435,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root=sys.argv[1], directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} @@ -445,7 +446,9 @@ labels = LABELS_DICT # Regenerate questions with real community labels (labels affect question phrasing) questions = suggest_questions(G, communities, labels) -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, sys.argv[1], suggested_questions=questions) +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") +Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") # Re-export so graph.json nodes carry the curated community_name (#2490). # Same extraction as Step 4, so the #479 shrink-guard passes on node count; # if it still refuses, surface the guard message - do not force past it. @@ -453,11 +456,8 @@ wrote = to_json(G, communities, 'graphify-out/graph.json', community_labels=labe if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') print('If this shrink is intentional (you deleted files), re-run a full build with --force.') - raise SystemExit(1) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") print('Report updated with community labels') -" "INPUT_PATH" +" ``` Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). @@ -472,14 +472,15 @@ If `--obsidian` was given: - If `--obsidian-dir ` was also given, pass it via `--dir`. Otherwise defaults to `graphify-out/obsidian`. ```bash -$(cat graphify-out/.graphify_python) -m graphify export obsidian -# Add `--dir ~/vaults/my-project` to use a custom directory. +graphify export obsidian +# or with custom dir: graphify export obsidian --dir ~/vaults/my-project ``` Generate the HTML graph (always, unless `--no-viz`): ```bash -$(cat graphify-out/.graphify_python) -m graphify export html # auto-aggregates to community view if graph > 5000 nodes +graphify export html # auto-aggregates to community view if graph > 5000 nodes +# or: graphify export html --no-viz ``` ### Steps 6b-8 - Wiki, Neo4j, FalkorDB, SVG, GraphML, MCP, benchmark (only on their flags) @@ -492,7 +493,7 @@ These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, ` ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from pathlib import Path from datetime import datetime, timezone from graphify.detect import save_manifest @@ -515,7 +516,7 @@ extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encod # types are gated on output. from graphify.cli import _stamped_manifest_files _corpus = detect.get('all_files') or detect['files'] -_manifest_files = _stamped_manifest_files(_corpus, extract, Path(sys.argv[1])) +_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) # Files dispatched this run (the changed subset) but NOT stamped above still carry # a stale semantic_hash from a prior run; clear it so detect_incremental re-queues # them instead of reading them as unchanged (#1948). @@ -527,7 +528,7 @@ _cleared = _dispatched - _stamped # files newly excluded since last run are dropped rather than masquerading as # deletions; untouched files' prior rows are still preserved (#1908). _scan = {f for fl in _corpus.values() for f in fl} -save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) # Update cumulative cost tracker input_tok = extract.get('input_tokens', 0) @@ -551,10 +552,10 @@ cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\" print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" "INPUT_PATH" +" rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null -rm -f graphify-out/needs_update 2>/dev/null || true +rm -f graphify-out/.needs_update 2>/dev/null || true ``` Replace INPUT_PATH with the actual path (same value used in Steps 4-5) so the manifest is relativized to the scan root. diff --git a/tools/skillgen/fragments/core/devin.md b/tools/skillgen/fragments/core/devin.md index 8740ca565a..f9be846cbf 100644 --- a/tools/skillgen/fragments/core/devin.md +++ b/tools/skillgen/fragments/core/devin.md @@ -114,12 +114,11 @@ If the import succeeds, print nothing and move straight to Step 2. ```bash $(cat graphify-out/.graphify_python) -c " import json -import sys from graphify.detect import detect from pathlib import Path -result = detect(Path(sys.argv[1])) +result = detect(Path('INPUT_PATH')) print(json.dumps(result)) -" "INPUT_PATH" > graphify-out/.graphify_detect.json +" > graphify-out/.graphify_detect.json ``` Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: @@ -147,11 +146,13 @@ Skip this step entirely if `detect` returned zero `video` files. Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. -**Strategy:** If an analysis file exists from a previous run, read its top god-node labels and write a one-sentence domain hint from them. Otherwise use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` Pass the chosen prompt to Whisper as its initial prompt. No separate API call needed. +**Strategy:** Read the god nodes from the detect output or analysis file. You are already a language model - write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. + +**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` **Step 1 - Write the Whisper prompt yourself.** -Read the top god-node labels from previous analysis when available, then compose a short domain hint sentence. With no previous analysis, use the generic fallback above. +Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: - Labels: `transformer, attention, encoder, decoder` -> `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` - Labels: `kubernetes, deployment, pod, helm` -> `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` @@ -490,7 +491,7 @@ wrote = to_json(G, communities, 'graphify-out/graph.json') if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (fewer nodes than the existing graph). Run a full rebuild to be safe.') raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, sys.argv[1], suggested_questions=questions) +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) Path('graphify-out/GRAPH_REPORT.md').write_text(report) analysis = { @@ -502,7 +503,7 @@ analysis = { } Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2)) print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" "INPUT_PATH" +" ``` If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. @@ -540,18 +541,17 @@ labels = LABELS_DICT # Regenerate questions with real community labels (labels affect question phrasing) questions = suggest_questions(G, communities, labels) -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, sys.argv[1], suggested_questions=questions) +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report) +Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()})) # Re-export so graph.json nodes carry the curated community_name (#2490). # Same extraction as Step 4, so the #479 shrink-guard passes on node count; # if it still refuses, surface the guard message - do not force past it. wrote = to_json(G, communities, 'graphify-out/graph.json', community_labels=labels) if not wrote: print('ERROR: refused to shrink graphify-out/graph.json (fewer nodes than the existing graph). Run a full rebuild to be safe.') - raise SystemExit(1) -Path('graphify-out/GRAPH_REPORT.md').write_text(report) -Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()})) print('Report updated with community labels') -" "INPUT_PATH" +" ``` Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). @@ -796,7 +796,6 @@ Print the output directly in chat. If `total_words <= 5000`, skip silently - the ```bash $(cat graphify-out/.graphify_python) -c " import json -import sys from pathlib import Path from datetime import datetime, timezone from graphify.detect import save_manifest @@ -807,13 +806,13 @@ extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) # Stamp only semantic files that produced output so a failed chunk is re-queued next run, not lost (#2015). from graphify.cli import _stamped_manifest_files _corpus = detect.get('all_files') or detect['files'] -_manifest_files = _stamped_manifest_files(_corpus, extract, Path(sys.argv[1])) +_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) _sem_types = ('document', 'paper', 'image') _dispatched = {f for t, fl in detect['files'].items() if t in _sem_types for f in fl} _stamped = {f for fl in _manifest_files.values() for f in fl} _cleared = _dispatched - _stamped _scan = {f for fl in _corpus.values() for f in fl} -save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) # Update cumulative cost tracker input_tok = extract.get('input_tokens', 0) @@ -837,9 +836,9 @@ cost_path.write_text(json.dumps(cost, indent=2)) print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" "INPUT_PATH" +" rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json graphify-out/.graphify_labels.json graphify-out/.graphify_incremental.json graphify-out/.graphify_transcripts.json graphify-out/.graphify_old.json; find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null -rm -f graphify-out/needs_update 2>/dev/null || true +rm -f graphify-out/.needs_update 2>/dev/null || true ``` Tell the user (omit the obsidian line unless --obsidian was given; omit the wiki line unless --wiki was given): @@ -904,7 +903,7 @@ import sys, json from graphify.detect import detect_incremental, save_manifest from pathlib import Path -result = detect_incremental(Path(sys.argv[1])) +result = detect_incremental(Path('INPUT_PATH')) new_total = result.get('new_total', 0) print(json.dumps(result, indent=2)) Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result)) @@ -916,7 +915,7 @@ if deleted: print(f'{len(deleted)} deleted file(s) to prune.') if new_total > 0: print(f'{new_total} new/changed file(s) to re-extract.') -" "INPUT_PATH" +" ``` If new files exist, first check whether all changed files are code files: @@ -959,13 +958,13 @@ $(cat graphify-out/.graphify_python) -c " import sys, json from graphify.build import build_from_json from graphify.export import to_json -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph import networkx as nx from pathlib import Path # Load existing graph existing_data = json.loads(Path('graphify-out/graph.json').read_text()) -G_existing = load_node_link_graph(existing_data) +G_existing = json_graph.node_link_graph(existing_data, edges='links') # Load new extraction new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) @@ -986,7 +985,7 @@ $(cat graphify-out/.graphify_python) -c " import json from graphify.analyze import graph_diff from graphify.build import build_from_json -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph import networkx as nx from pathlib import Path @@ -995,7 +994,7 @@ new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text() G_new = build_from_json(new_extract, directed=IS_DIRECTED) if old_data: - G_old = load_node_link_graph(old_data) + G_old = json_graph.node_link_graph(old_data, edges='links') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: @@ -1021,12 +1020,12 @@ from graphify.cluster import cluster, score_all from graphify.analyze import god_nodes, surprising_connections from graphify.report import generate from graphify.export import to_json -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph import networkx as nx from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text()) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') detection = {'total_files': 0, 'total_words': 99999, 'needs_graph': True, 'warning': None, 'files': {'code': [], 'document': [], 'paper': []}} @@ -1090,12 +1089,12 @@ Load `graphify-out/graph.json`, then: ```bash $(cat graphify-out/.graphify_python) -c " import sys, json -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph import networkx as nx from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text()) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' @@ -1202,11 +1201,11 @@ if not Path('graphify-out/graph.json').exists(): $(cat graphify-out/.graphify_python) -c " import json, sys import networkx as nx -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text()) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') a_term = 'NODE_A' b_term = 'NODE_B' @@ -1274,11 +1273,11 @@ if not Path('graphify-out/graph.json').exists(): $(cat graphify-out/.graphify_python) -c " import json, sys import networkx as nx -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text()) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') term = 'NODE_NAME' term_lower = term.lower() @@ -1332,7 +1331,7 @@ from graphify.ingest import ingest from pathlib import Path try: - out = ingest(sys.argv[1], Path('./raw'), author=sys.argv[2] or None, contributor=sys.argv[3] or None) + out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') print(f'Saved to {out}') except ValueError as e: print(f'error: {e}', file=sys.stderr) @@ -1340,10 +1339,10 @@ except ValueError as e: except RuntimeError as e: print(f'error: {e}', file=sys.stderr) sys.exit(1) -" "URL" "AUTHOR" "CONTRIBUTOR" +" ``` -Replace `URL` with the actual URL and pass the user's `AUTHOR` or `CONTRIBUTOR` when provided; use an empty string for either omitted value. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. +Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. Supported URL types (auto-detected): - Twitter/X -> fetched via oEmbed, saved as `.md` with tweet text and author @@ -1359,7 +1358,7 @@ Supported URL types (auto-detected): Start a background watcher that monitors a folder and auto-updates the graph when files change. ```bash -python3 -m graphify.watch "INPUT_PATH" --debounce 3 +python3 -m graphify.watch INPUT_PATH --debounce 3 ``` Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: diff --git a/tools/skillgen/fragments/query-stub/default.md b/tools/skillgen/fragments/query-stub/default.md index 54e1999630..696796ec5c 100644 --- a/tools/skillgen/fragments/query-stub/default.md +++ b/tools/skillgen/fragments/query-stub/default.md @@ -1,7 +1,7 @@ When `graphify-out/graph.json` already exists and the user asks a question about the corpus, answer from the graph rather than rebuilding it: ```bash -$(cat graphify-out/.graphify_python) -m graphify query "" +graphify query "" ``` Before traversal, expand the question against the graph's own vocabulary so a wording mismatch does not collapse the answer to noise. If the `graphify query` CLI is unavailable, fall back to an inline NetworkX traversal of `graphify-out/graph.json`. Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. For that vocab-expansion step, the BFS/DFS traversal modes, the `--budget` cap, the NetworkX fallback, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`. diff --git a/tools/skillgen/fragments/references/query/default.md b/tools/skillgen/fragments/references/query/default.md index 6dd627863c..56565eb782 100644 --- a/tools/skillgen/fragments/references/query/default.md +++ b/tools/skillgen/fragments/references/query/default.md @@ -79,12 +79,12 @@ If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal ```bash $(cat graphify-out/.graphify_python) -c " import sys, json -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph import networkx as nx from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' @@ -197,11 +197,11 @@ If the CLI is unavailable, run it inline: $(cat graphify-out/.graphify_python) -c " import json, sys import networkx as nx -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') a_term = 'NODE_A' b_term = 'NODE_B' @@ -265,11 +265,11 @@ If the CLI is unavailable, run it inline: $(cat graphify-out/.graphify_python) -c " import json, sys import networkx as nx -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph from pathlib import Path data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = load_node_link_graph(data) +G = json_graph.node_link_graph(data, edges='links') term = 'NODE_NAME' term_lower = term.lower() diff --git a/tools/skillgen/fragments/references/shared/add-watch.md b/tools/skillgen/fragments/references/shared/add-watch.md index fcbe0cd488..77844343e1 100644 --- a/tools/skillgen/fragments/references/shared/add-watch.md +++ b/tools/skillgen/fragments/references/shared/add-watch.md @@ -13,7 +13,7 @@ from graphify.ingest import ingest from pathlib import Path try: - out = ingest(sys.argv[1], Path('./raw'), author=sys.argv[2] or None, contributor=sys.argv[3] or None) + out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') print(f'Saved to {out}') except ValueError as e: print(f'error: {e}', file=sys.stderr) @@ -21,10 +21,10 @@ except ValueError as e: except RuntimeError as e: print(f'error: {e}', file=sys.stderr) sys.exit(1) -" "URL" "AUTHOR" "CONTRIBUTOR" +" ``` -Replace `URL` with the actual URL and pass the user's `AUTHOR` or `CONTRIBUTOR` when provided; use an empty string for either omitted value. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on the scan root recorded in `graphify-out/.graphify_root` to merge the new file into the existing graph. +Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. Supported URL types (auto-detected): - YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`) @@ -41,7 +41,7 @@ Supported URL types (auto-detected): Start a background watcher that monitors a folder and auto-updates the graph when files change. ```bash -$(cat graphify-out/.graphify_python) -m graphify.watch "INPUT_PATH" --debounce 3 +$(cat graphify-out/.graphify_python) -m graphify.watch INPUT_PATH --debounce 3 ``` Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: diff --git a/tools/skillgen/fragments/references/shared/exports.md b/tools/skillgen/fragments/references/shared/exports.md index 89ce996fb7..242ff868e0 100644 --- a/tools/skillgen/fragments/references/shared/exports.md +++ b/tools/skillgen/fragments/references/shared/exports.md @@ -20,27 +20,13 @@ graphify export wiki graphify export neo4j ``` -**If `--neo4j-push `** - push directly to a running Neo4j instance. Do not run these blocks through an agent tool: its shell is non-interactive and cannot read a hidden password. Show the block for the user's shell, ask them to run it in their own terminal, and wait for confirmation before continuing. Never ask them to paste the password into chat. - -POSIX shell: +**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: ```bash -NEO4J_PASSWORD="$("$(cat graphify-out/.graphify_python)" -c 'import getpass; print(getpass.getpass("Neo4j password: "))')" -export NEO4J_PASSWORD -graphify export neo4j --push bolt://localhost:7687 --user neo4j -unset NEO4J_PASSWORD -``` - -PowerShell: - -```powershell -$credential = Read-Host "Neo4j password" -AsSecureString -$env:NEO4J_PASSWORD = [System.Net.NetworkCredential]::new("", $credential).Password -graphify export neo4j --push bolt://localhost:7687 --user neo4j -Remove-Item Env:NEO4J_PASSWORD +graphify export neo4j --push bolt://localhost:7687 --user neo4j --password PASSWORD ``` -Default URI is `bolt://localhost:7687`, default user is `neo4j`. The command reads the password from `NEO4J_PASSWORD`. Uses MERGE - safe to re-run without creating duplicates. +Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. ### Step 7a - FalkorDB export (only if --falkordb or --falkordb-push flag) diff --git a/tools/skillgen/fragments/references/shared/extraction-spec-compact.md b/tools/skillgen/fragments/references/shared/extraction-spec-compact.md index 8aaa23729b..4b278b28d3 100644 --- a/tools/skillgen/fragments/references/shared/extraction-spec-compact.md +++ b/tools/skillgen/fragments/references/shared/extraction-spec-compact.md @@ -25,7 +25,7 @@ Rules: Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format `{stem}_{entity}` where stem is the full repo-relative path with the extension dropped, every segment joined with `_` (each lowercased with non-alphanumeric chars replaced by `_`) and entity is the symbol name similarly normalized. Use every directory level, not just the immediate parent. `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`. Top-level files use just the filename stem. This must match the AST extractor's ID. Never append chunk or sequence suffixes — IDs must be deterministic from the label alone. Output exactly this JSON (no other text): -{"nodes":[{"id":"src_auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} +{"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} source_file RULE: set source_file to the FILE_LIST path for that file VERBATIM (absolute, no shortening to basename, no re-relativizing, no separator change). Keeps full build and --update on one base so build_merge's replace matches instead of duplicating. ``` diff --git a/tools/skillgen/fragments/references/shared/extraction-spec.md b/tools/skillgen/fragments/references/shared/extraction-spec.md index a5a12a575e..388df7674f 100644 --- a/tools/skillgen/fragments/references/shared/extraction-spec.md +++ b/tools/skillgen/fragments/references/shared/extraction-spec.md @@ -61,7 +61,7 @@ confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a d Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is the **full repo-relative path with the extension dropped**, every path segment kept and joined with `_` (each segment lowercased with non-alphanumeric chars replaced by `_`), and entity is the symbol name similarly normalized. Use every directory level, not just the immediate parent — this keeps same-named files in different directories distinct. Examples: `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `lib_utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`; `docs/v1/api/README.md` + `getUser` → `docs_v1_api_readme_getuser`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or only the immediate parent (e.g., `auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project built under the old immediate-parent format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. Generate the extraction JSON matching this schema exactly: -{"nodes":[{"id":"src_auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} +{"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. diff --git a/tools/skillgen/fragments/references/shared/transcribe.md b/tools/skillgen/fragments/references/shared/transcribe.md index bbe8b93e45..b967f83799 100644 --- a/tools/skillgen/fragments/references/shared/transcribe.md +++ b/tools/skillgen/fragments/references/shared/transcribe.md @@ -8,11 +8,13 @@ Skip this step entirely if `detect` returned zero `video` files. Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. -**Strategy:** If `graphify-out/.graphify_analysis.json` exists from a previous run, read its top god-node labels and write a one-sentence domain hint from them. Otherwise use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` Pass the chosen prompt to Whisper as its initial prompt. No separate API call needed. +**Strategy:** Read the god nodes from `graphify-out/.graphify_detect.json` (or the analysis file if it exists from a previous run). You are already a language model — write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. + +**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` **Step 1 - Write the Whisper prompt yourself.** -Read the top god-node labels from previous analysis when available, then compose a short domain hint sentence. With no previous analysis, use the generic fallback above. +Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: - Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` - Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` diff --git a/tools/skillgen/fragments/references/shared/update.md b/tools/skillgen/fragments/references/shared/update.md index 058ca8e6d2..3632fd4126 100644 --- a/tools/skillgen/fragments/references/shared/update.md +++ b/tools/skillgen/fragments/references/shared/update.md @@ -12,7 +12,7 @@ import sys, json from graphify.detect import detect_incremental, save_manifest from pathlib import Path -result = detect_incremental(Path(sys.argv[1])) +result = detect_incremental(Path('INPUT_PATH')) new_total = result.get('new_total', 0) print(json.dumps(result, indent=2, ensure_ascii=False)) Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") @@ -24,7 +24,7 @@ if deleted: print(f'{len(deleted)} deleted file(s) to prune.') if new_total > 0: print(f'{new_total} new/changed file(s) to re-extract.') -" "INPUT_PATH" +" ``` Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: @@ -84,7 +84,7 @@ Then: ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json +import json from pathlib import Path from graphify.build import build_merge from graphify.detect import save_manifest @@ -113,7 +113,7 @@ G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, - root=sys.argv[1], + root='INPUT_PATH', directed=IS_DIRECTED, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -150,7 +150,7 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) # is lost forever (#2015). Mirrors the library extract path # (cli._stamped_manifest_files + clear_semantic + scan_corpus). from graphify.cli import _stamped_manifest_files -_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path(sys.argv[1])) +_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) # Changed semantic files dispatched this run but NOT stamped had their chunk fail # or be omitted; clear any stale semantic_hash so they are re-queued (#1948). _sem_types = ('document', 'paper', 'image') @@ -160,9 +160,9 @@ _cleared = _dispatched - _stamped # scan_corpus = the RAW full corpus so in-root files newly excluded since last run # are dropped rather than masquerading as deletions; untouched rows preserved (#1908). _scan = {f for fl in incremental['files'].values() for f in fl} -save_manifest(_manifest_files, root=sys.argv[1], scan_corpus=_scan, clear_semantic=_cleared or None) +save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) print('[graphify update] Manifest saved.') -" "INPUT_PATH" +" ``` Then run Steps 4–8 on the merged graph as normal. @@ -174,7 +174,7 @@ $(cat graphify-out/.graphify_python) -c " import json from graphify.analyze import graph_diff from graphify.build import build_from_json -from graphify.paths import load_node_link_graph +from networkx.readwrite import json_graph import networkx as nx from pathlib import Path @@ -184,7 +184,7 @@ new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(e G_new = build_from_json(new_extract, directed=IS_DIRECTED) if old_data: - G_old = load_node_link_graph(old_data) + G_old = json_graph.node_link_graph(old_data, edges='links') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: diff --git a/tools/skillgen/fragments/shell/posix.md b/tools/skillgen/fragments/shell/posix.md index 54ae63f87c..3534417d23 100644 --- a/tools/skillgen/fragments/shell/posix.md +++ b/tools/skillgen/fragments/shell/posix.md @@ -31,7 +31,7 @@ fi mkdir -p graphify-out "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" # Save scan root so `graphify update` (no args) knows where to look next time -echo "$(cd "INPUT_PATH" && pwd)" > graphify-out/.graphify_root +echo "$(cd INPUT_PATH && pwd)" > graphify-out/.graphify_root ``` If the import succeeds, print nothing and move straight to Step 2. diff --git a/tools/skillgen/fragments/shell/powershell.md b/tools/skillgen/fragments/shell/powershell.md index 62caf5c9ec..71e493cf8d 100644 --- a/tools/skillgen/fragments/shell/powershell.md +++ b/tools/skillgen/fragments/shell/powershell.md @@ -58,7 +58,7 @@ if (-not $GRAPHIFY_PYTHON) { $Utf8NoBom = New-Object System.Text.UTF8Encoding $false [System.IO.File]::WriteAllText((Join-Path $PWD 'graphify-out\.graphify_python'), [string]$GRAPHIFY_PYTHON, $Utf8NoBom) # Save scan root so `graphify update` (no args) knows where to look next time -[System.IO.File]::WriteAllText((Join-Path $PWD 'graphify-out\.graphify_root'), (Resolve-Path "INPUT_PATH").Path, $Utf8NoBom) +[System.IO.File]::WriteAllText((Join-Path $PWD 'graphify-out\.graphify_root'), (Resolve-Path INPUT_PATH).Path, $Utf8NoBom) ``` If the import succeeds, print nothing and move straight to Step 2. diff --git a/tools/skillgen/gen.py b/tools/skillgen/gen.py index a7c6dc5222..09e19ede00 100644 --- a/tools/skillgen/gen.py +++ b/tools/skillgen/gen.py @@ -384,14 +384,6 @@ def _render_frontmatter(platform: Platform) -> str: _PY_INVOKE_POSIX = '$(cat graphify-out/.graphify_python) -c "' _PY_INVOKE_PS_OPEN = "@'" _PY_INVOKE_PS_CLOSE = "'@ | & (Get-Content graphify-out\\.graphify_python) -" -_PY_CLOSE_TRANSLATIONS = { - '"': _PY_INVOKE_PS_CLOSE, - '" "INPUT_PATH"': _PY_INVOKE_PS_CLOSE + ' "INPUT_PATH"', - '" "INPUT_PATH" "SPEC_PATH"': _PY_INVOKE_PS_CLOSE + ' "INPUT_PATH" "SPEC_PATH"', - '" "URL" "AUTHOR" "CONTRIBUTOR"': ( - _PY_INVOKE_PS_CLOSE + ' "URL" "AUTHOR" "CONTRIBUTOR"' - ), -} _MKDIR_POSIX = "mkdir -p graphify-out" _MKDIR_PS = "New-Item -ItemType Directory -Force -Path graphify-out | Out-Null" _FIND_CHUNKS_POSIX = "find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null" @@ -402,8 +394,6 @@ def _render_frontmatter(platform: Platform) -> str: # Bash-only tokens that must never survive in a powershell-shell render. _POWERSHELL_BANNED_TOKENS = ("$(cat ", "rm -f ", "2>/dev/null", "```bash") -_GRAPHIFY_INVOKE_POSIX = "$(cat graphify-out/.graphify_python) -m graphify " -_GRAPHIFY_INVOKE_PS = "& (Get-Content graphify-out\\.graphify_python) -m graphify " def _unescape_bash_dq(line: str) -> str: @@ -444,8 +434,8 @@ def _translate_bash_block(lines: list[str]) -> list[str]: in_py = False for line in lines: if in_py: - if line in _PY_CLOSE_TRANSLATIONS: - out.append(_PY_CLOSE_TRANSLATIONS[line]) + if line == '"': + out.append(_PY_INVOKE_PS_CLOSE) in_py = False else: out.append(_unescape_bash_dq(line)) @@ -458,10 +448,8 @@ def _translate_bash_block(lines: list[str]) -> list[str]: out.append(_FIND_CHUNKS_PS) elif line.strip().startswith("rm -f "): out.append(_rm_to_remove_item(line)) - elif line.startswith(_GRAPHIFY_INVOKE_POSIX): - out.append(_GRAPHIFY_INVOKE_PS + line.removeprefix(_GRAPHIFY_INVOKE_POSIX)) elif not line.strip() or line.lstrip().startswith("#") or line.startswith("graphify "): - out.append(line) # blanks, comments, and explicit installer commands are shell-neutral + out.append(line) # blank lines, comments, and graphify CLI calls are shell-neutral else: raise ValueError(f"cannot translate bash line to PowerShell: {line!r}") if in_py: @@ -1154,78 +1142,6 @@ def _is_community_label_export_fix_line(line: str) -> bool: ) -def _is_needs_update_cleanup_fix_line(line: str) -> bool: - """Whether cleanup targets the watcher's actual ``needs_update`` flag.""" - return "rm -f graphify-out/" in line and "needs_update" in line - - -def _is_graph_loader_fix_line(line: str) -> bool: - """Whether a monolith delegates graph.json compatibility to the shared loader.""" - return line.strip() in { - "from networkx.readwrite import json_graph", - "from graphify.paths import load_node_link_graph", - "G_existing = json_graph.node_link_graph(existing_data, edges='links')", - "G_existing = load_node_link_graph(existing_data)", - "G_old = json_graph.node_link_graph(old_data, edges='links')", - "G_old = load_node_link_graph(old_data)", - "G = json_graph.node_link_graph(data, edges='links')", - "G = load_node_link_graph(data)", - } - - -def _is_watch_path_quote_fix_line(line: str) -> bool: - """Whether a line is the legacy or safely quoted monolith watch command.""" - return line.strip() in { - "python3 -m graphify.watch INPUT_PATH --debounce 3", - 'python3 -m graphify.watch "INPUT_PATH" --debounce 3', - } - - -def _is_python_path_argument_fix_line(line: str) -> bool: - """Whether a monolith passes INPUT_PATH as argv instead of Python source.""" - return line.strip() in { - "import sys", - "result = detect(Path('INPUT_PATH'))", - "result = detect(Path(sys.argv[1]))", - "result = detect_incremental(Path('INPUT_PATH'))", - "result = detect_incremental(Path(sys.argv[1]))", - '"', - '" "INPUT_PATH"', - '" > .graphify_detect.json', - '" "INPUT_PATH" > .graphify_detect.json', - '" > graphify-out/.graphify_detect.json', - '" "INPUT_PATH" > graphify-out/.graphify_detect.json', - "report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions)", - "report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, sys.argv[1], suggested_questions=questions)", - "report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions)", - "report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, sys.argv[1], suggested_questions=questions)", - } - - -def _is_ingest_argument_fix_line(line: str) -> bool: - """Whether URL metadata is passed as argv instead of Python source.""" - return line.strip() in { - "out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR')", - "out = ingest(sys.argv[1], Path('./raw'), author=sys.argv[2] or None, contributor=sys.argv[3] or None)", - '" "URL" "AUTHOR" "CONTRIBUTOR"', - "Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph.", - "Replace `URL` with the actual URL and pass the user's `AUTHOR` or `CONTRIBUTOR` when provided; use an empty string for either omitted value. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph.", - "Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph.", - "Replace `URL` with the actual URL and pass the user's `AUTHOR` or `CONTRIBUTOR` when provided; use an empty string for either omitted value. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph.", - } - - -def _is_transcription_analysis_fix_line(line: str) -> bool: - """Whether transcription uses prior analysis rather than detect's nonexistent god nodes.""" - return line.strip() in { - '**Strategy:** Read the god nodes from the detect output or analysis file. You are already a language model - write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed.', - '**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."`', - 'Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example:', - '**Strategy:** If an analysis file exists from a previous run, read its top god-node labels and write a one-sentence domain hint from them. Otherwise use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` Pass the chosen prompt to Whisper as its initial prompt. No separate API call needed.', - 'Read the top god-node labels from previous analysis when available, then compose a short domain hint sentence. With no previous analysis, use the generic fallback above.', - } - - # Every line that may differ between a rendered monolith and its pristine v8 # baseline. Each predicate documents one sanctioned change-class; a blank line is # allowed because the multi-line fix blocks insert spacing. Anything else failing @@ -1247,12 +1163,6 @@ def _is_transcription_analysis_fix_line(line: str) -> bool: _is_uv_from_interpreter_fix_line, _is_semantic_cache_scope_fix_line, _is_community_label_export_fix_line, - _is_needs_update_cleanup_fix_line, - _is_graph_loader_fix_line, - _is_watch_path_quote_fix_line, - _is_python_path_argument_fix_line, - _is_ingest_argument_fix_line, - _is_transcription_analysis_fix_line, )