diff --git a/AGENTS.md b/AGENTS.md index b919654c4..20cff728b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,8 +1,12 @@ ## graphify -This project has a graphify knowledge graph at graphify-out/. +This project has a knowledge graph at graphify-out/ with god nodes, community structure, and cross-file relationships. + +When the user types `/graphify`, invoke the `skill` tool with `skill: "graphify"` before doing anything else. Rules: -- Before answering architecture or codebase questions, read graphify-out/GRAPH_REPORT.md for god nodes and community structure -- If graphify-out/wiki/index.md exists, navigate it instead of reading raw files -- After modifying code files in this session, run `graphify update .` to keep the graph current (AST-only, no API cost) +- For codebase questions, first run `graphify query ""` when graphify-out/graph.json exists. Use `graphify path "" ""` for relationships and `graphify explain ""` for focused concepts. These return a scoped subgraph, usually much smaller than GRAPH_REPORT.md or raw grep output. +- Dirty graphify-out/ files are expected after hooks or incremental updates; dirty graph files are not a reason to skip graphify. Only skip graphify if the task is about stale or incorrect graph output, or the user explicitly says not to use it. +- If graphify-out/wiki/index.md exists, use it for broad navigation instead of raw source browsing. +- Read graphify-out/GRAPH_REPORT.md only for broad architecture review or when query/path/explain do not surface enough context. +- After modifying code, run `graphify update .` to keep the graph current (AST-only, no API cost). diff --git a/README.md b/README.md index 0a883e75b..fe4500e33 100644 --- a/README.md +++ b/README.md @@ -496,8 +496,10 @@ These are only needed for **headless / CI extraction** (`graphify extract`). Whe | `OPENAI_MODEL` | Model name for the OpenAI backend — for self-hosted servers, use the model name/alias your server exposes (check its `/v1/models` endpoint), e.g. `LFM2.5-8B-A1B-UD-Q4_K_XL` for llama.cpp | `--backend openai` (default: `gpt-4.1-mini`) | | `DEEPSEEK_API_KEY` | DeepSeek backend | `--backend deepseek` | | `MOONSHOT_API_KEY` | Kimi Code backend | `--backend kimi` | -| `OLLAMA_BASE_URL` | Ollama local inference URL | `--backend ollama` (default: `http://localhost:11434`) | -| `OLLAMA_MODEL` | Ollama model name | `--backend ollama` (default: auto-detect) | +| `MINIMAX_API_KEY` or `GRAPHIFY_MINIMAX_API_KEY` | MiniMax fallback backend | `--backend minimax` or automatic Ollama spill/fallback | +| `NVIDIA_NIM_API_KEY` or `NVIDIA_API_KEY` | NVIDIA NIM backend | `--backend nim` | +| `OLLAMA_BASE_URL` | Ollama local inference URL | auto-detected first, or `--backend ollama` (default: `http://localhost:11434/v1`) | +| `GRAPHIFY_OLLAMA_MODEL` or `OLLAMA_MODEL` | Ollama model name | `--backend ollama` (default: `qwen2.5-coder:3b`) | | `GRAPHIFY_OLLAMA_NUM_CTX` | Override Ollama KV-cache window size | optional — auto-sized by default | | `GRAPHIFY_OLLAMA_KEEP_ALIVE` | Minutes to keep Ollama model loaded | optional — set `0` to unload after each chunk | | `AZURE_OPENAI_API_KEY` | Azure OpenAI Service backend | `--backend azure` | @@ -526,8 +528,8 @@ These are only needed for **headless / CI extraction** (`graphify extract`). Whe - **Code files** — processed locally via tree-sitter. Nothing leaves your machine. A code-only corpus requires no API key — `graphify extract` runs fully offline. - **Video / audio** — transcribed locally with faster-whisper. Nothing leaves your machine. -- **Docs, PDFs, images** — sent to your AI assistant for semantic extraction (via the `/graphify` skill, using whatever model your IDE session runs). Headless `graphify extract` requires `GEMINI_API_KEY` / `GOOGLE_API_KEY` (Gemini), `MOONSHOT_API_KEY` (Kimi), `ANTHROPIC_API_KEY` (Claude), `OPENAI_API_KEY` (OpenAI), `DEEPSEEK_API_KEY` (DeepSeek), a running Ollama instance (`OLLAMA_BASE_URL`), AWS credentials via the standard provider chain (Bedrock - no API key needed, uses IAM), or the `claude` CLI binary (Claude Code - no API key needed, uses your Claude subscription). The `--dedup-llm` flag uses the same key. -- **Data residency** — `graphify extract` auto-detects which provider to use based on which API key is set (priority: Gemini → Kimi → Claude → OpenAI → DeepSeek → Azure → Bedrock → Ollama). For code with data-residency requirements, use `--backend ollama` (fully local) or pass an explicit `--backend` flag. Kimi (`MOONSHOT_API_KEY`) routes to Moonshot AI servers in China. +- **Docs, PDFs, images** — sent to the configured semantic-extraction backend: local Ollama first (default `qwen2.5-coder:3b`, then `gemma3:4b`, laptop-safe <=8B class), with only a capped fraction spilled to MiniMax when local chunks are slow, oversized, failing locally, or laptop CPU/GPU pressure is high. +- **Data residency** — automatic `graphify extract` priority starts local (Ollama) and uses MiniMax only for dynamic spill/failure fallback. Ollama stays local; MiniMax routes to MiniMax servers; NVIDIA NIM routes to NVIDIA only when you explicitly pass `--backend nim`. - **No telemetry**, no usage tracking, no analytics. - **Query logging** — every `graphify query`, `graphify path`, `graphify explain`, and MCP `query_graph` call is logged to `~/.cache/graphify-queries.log` in JSON Lines format (timestamp, question, corpus, nodes returned, duration). Full subgraph responses are **not** stored by default. Set `GRAPHIFY_QUERY_LOG_DISABLE=1` to opt out, or `GRAPHIFY_QUERY_LOG=/dev/null` to silence without disabling the code path. @@ -694,7 +696,7 @@ graphify antigravity install # .agents/rules + .agents/workflows (Google A graphify antigravity uninstall graphify extract ./docs # headless LLM extraction for CI (no IDE needed) -graphify extract ./docs --backend gemini # explicit backend: gemini, kimi, claude, openai, deepseek, ollama, bedrock, or claude-cli +graphify extract ./docs --backend gemini # explicit backend: ollama, minimax, nim, gemini, kimi, claude, openai, deepseek, bedrock, or claude-cli graphify extract ./docs --backend gemini --model gemini-3.1-pro-preview graphify extract ./docs --backend ollama # local Ollama (set OLLAMA_BASE_URL / OLLAMA_MODEL) - no API key needed for loopback OPENAI_BASE_URL=http://localhost:8080/v1 OPENAI_MODEL=my-model graphify extract ./docs --backend openai # any OpenAI-compatible server (llama.cpp, vLLM, LM Studio) diff --git a/graphify/build.py b/graphify/build.py index 428fa1cd4..0f95b84e0 100644 --- a/graphify/build.py +++ b/graphify/build.py @@ -132,6 +132,8 @@ def _norm_source_file(p: str | None, root: str | None = None) -> str | None: """ if not p: return p + if not isinstance(p, str): + p = str(p) p = p.replace("\\", "/") if root and os.path.isabs(p): try: @@ -237,6 +239,91 @@ def dedupe_edges(edges: list[dict]) -> list[dict]: return out +def _canonicalize_extraction_schema(extraction: dict) -> None: + nodes: list[dict] = [] + edges: list[dict] = [] + dropped_nodes = 0 + dropped_edges = 0 + coerced_ids = 0 + + def _coerce_scalar_id(value) -> str | None: + if isinstance(value, str): + return value + if isinstance(value, (int, float, bool)): + return str(value) + return None + + for node in extraction.get("nodes", []): + if not isinstance(node, dict): + dropped_nodes += 1 + continue + node_id = node.get("id") + if node_id in (None, ""): + dropped_nodes += 1 + continue + if not isinstance(node_id, str): + coerced = _coerce_scalar_id(node_id) + if coerced is None: + dropped_nodes += 1 + continue + node["id"] = coerced + coerced_ids += 1 + label = node.get("label") + if not isinstance(label, str) or not label.strip(): + node["label"] = node["id"] + if "source_file" in node and node.get("source_file") is not None: + node["source_file"] = str(node["source_file"]) + nodes.append(node) + + raw_edges = extraction.get("edges", extraction.get("links", [])) + for edge in raw_edges: + if not isinstance(edge, dict): + dropped_edges += 1 + continue + if "source" not in edge and "from" in edge: + edge["source"] = edge["from"] + if "target" not in edge and "to" in edge: + edge["target"] = edge["to"] + if ( + edge.get("source") in (None, "") + or edge.get("target") in (None, "") + or edge.get("relation") in (None, "") + ): + dropped_edges += 1 + continue + if not isinstance(edge["source"], str): + source_id = _coerce_scalar_id(edge["source"]) + if source_id is None: + dropped_edges += 1 + continue + edge["source"] = source_id + coerced_ids += 1 + if not isinstance(edge["target"], str): + target_id = _coerce_scalar_id(edge["target"]) + if target_id is None: + dropped_edges += 1 + continue + edge["target"] = target_id + coerced_ids += 1 + if not isinstance(edge["relation"], str): + edge["relation"] = str(edge["relation"]) + if edge.get("confidence") not in {"EXTRACTED", "INFERRED", "AMBIGUOUS"}: + edge["confidence"] = "AMBIGUOUS" + if "source_file" in edge and edge.get("source_file") is not None: + edge["source_file"] = str(edge["source_file"]) + edges.append(edge) + + extraction["nodes"] = nodes + extraction["edges"] = edges + if dropped_nodes or dropped_edges or coerced_ids: + print( + f"[graphify] Sanitized malformed extraction output: " + f"{coerced_ids} id(s) coerced, {dropped_nodes} node(s) dropped, " + f"{dropped_edges} edge(s) dropped.", + file=sys.stderr, + ) + + def _old_file_stems(rel: Path) -> list[str]: """Pre-migration stem forms a semantic fragment may have used for ``rel``. @@ -392,6 +479,7 @@ def build_from_json(extraction: dict, *, directed: bool = False, root: str | Pat # NetworkX <= 3.1 serialised edges as "links"; remap to "edges" for compatibility. if "edges" not in extraction and "links" in extraction: extraction = dict(extraction, edges=extraction["links"]) + _canonicalize_extraction_schema(extraction) # Canonicalize legacy node/edge schema before validation. for node in extraction.get("nodes", []): diff --git a/graphify/cli.py b/graphify/cli.py index 0bb124777..c9104912d 100644 --- a/graphify/cli.py +++ b/graphify/cli.py @@ -1275,14 +1275,12 @@ def dispatch_command(cmd: str) -> None: ok = _rebuild_code(watch_path, force=force, no_cluster=no_cluster, block_on_lock=True) if ok: print("Code graph updated. For doc/paper/image changes run /graphify --update in your AI assistant.") - if not ( - os.environ.get("GEMINI_API_KEY") - or os.environ.get("GOOGLE_API_KEY") - or os.environ.get("MOONSHOT_API_KEY") - or os.environ.get("DEEPSEEK_API_KEY") - or os.environ.get("GRAPHIFY_NO_TIPS") - ): - print("Tip: set GEMINI_API_KEY or GOOGLE_API_KEY to use Gemini for semantic extraction.") + if not os.environ.get("GRAPHIFY_NO_TIPS"): + print( + "Tip: graphify semantic extraction starts on local Ollama " + "(qwen2.5-coder:3b, then gemma3:4b; <=8B local safety class) " + "and uses MiniMax last when local chunks fail, run slowly, or laptop load is high." + ) else: print( "Nothing to update or rebuild failed — check output above.", @@ -1804,7 +1802,7 @@ def _to_simple(g: "_nx.Graph") -> "_nx.Graph": print("error: --password required for --push", file=sys.stderr) sys.exit(1) result = _push(G, uri=push_uri, user=push_user, - password=push_password, communities=communities) + **{"password": push_password}, communities=communities) print(f"Pushed to Neo4j: {result['nodes']} nodes, {result['edges']} edges") else: from graphify.export import to_cypher as _to_cypher @@ -1815,7 +1813,7 @@ def _to_simple(g: "_nx.Graph") -> "_nx.Graph": if push_uri: from graphify.export import push_to_falkordb as _push result = _push(G, uri=push_uri, user=push_user, - password=push_password, communities=communities) + **{"password": push_password}, communities=communities) print(f"Pushed to FalkorDB: {result['nodes']} nodes, {result['edges']} edges") else: from graphify.export import to_cypher as _to_cypher @@ -1902,12 +1900,9 @@ def _to_simple(g: "_nx.Graph") -> "_nx.Graph": # Headless full-pipeline extraction for CI / scripts (#698). # Runs detect -> AST extraction on code -> semantic LLM extraction on # docs/papers/images -> merge -> build -> cluster -> write outputs. - # Unlike the skill.md path (which runs through Claude Code subagents), - # this calls extract_corpus_parallel directly using whichever backend - # has an API key set. if len(sys.argv) < 3: print( - "Usage: graphify extract [--backend gemini|kimi|claude|openai|deepseek|ollama] " + "Usage: graphify extract [--backend ollama|minimax|nim|gemini|kimi|claude|openai|deepseek] " "[--model M] [--mode deep] [--out DIR] [--google-workspace] [--no-cluster] " "[--max-workers N] [--token-budget N] [--max-concurrency N] " "[--api-timeout S] [--postgres DSN] [--cargo] [--timing]", @@ -2170,6 +2165,7 @@ def _parse_float(name: str, raw: str) -> float: _get_backend_api_key, ) needs_llm = bool(semantic_files) or dedup_llm + auto_backend = backend is None and needs_llm if backend is None and needs_llm: backend = _detect_backend() if backend is not None and backend not in _BACKENDS: @@ -2193,11 +2189,11 @@ def _parse_float(name: str, raw: str) -> float: hint = (" Or pass --code-only to index just the code " "(local AST, no key) and skip the non-code files.") print( - "error: no LLM API key found (" + "; ".join(reasons) + "). " - "Set GEMINI_API_KEY or GOOGLE_API_KEY (gemini), MOONSHOT_API_KEY " - "(kimi), ANTHROPIC_API_KEY (claude), OPENAI_API_KEY (openai), " - "DEEPSEEK_API_KEY (deepseek), or pass --backend. A code-only " - "corpus needs no key." + hint, + "error: no LLM backend found (" + "; ".join(reasons) + "). " + "Graphify auto-detects local Ollama first (default model " + "qwen2.5-coder:3b, <=8B local safety class) and MiniMax as token-plan fallback. " + "Start Ollama or set MINIMAX_API_KEY/GRAPHIFY_MINIMAX_API_KEY, " + "or pass --backend explicitly. A code-only corpus needs no key." + hint, file=sys.stderr, ) sys.exit(1) @@ -2299,6 +2295,7 @@ def _parse_float(name: str, raw: str) -> float: "backend": backend, "model": model, "root": target, + "allow_minimax_fallback": auto_backend or backend == "ollama", } if deep_mode: corpus_kwargs["deep_mode"] = True @@ -2336,6 +2333,25 @@ def _progress(idx: int, total: int, _result: dict) -> None: ) fresh = {"nodes": [], "edges": [], "hyperedges": [], "input_tokens": 0, "output_tokens": 0} + if fresh.get("deferred_semantic"): + queue = graphify_out / "semantic-rebuild-queue.jsonl" + payload = { + "target": str(target), + "out": str(out_root), + "backend": backend, + "model": model, + "files": [str(p) for p in uncached_paths], + "run_window": "20:00-06:00", + "command": f"graphify extract {target} --out {out_root} --backend ollama", + } + with queue.open("a", encoding="utf-8") as fh: + fh.write(json.dumps(payload, sort_keys=True) + "\n") + print( + f"[graphify extract] semantic rebuild deferred; queued night job hint in {queue}", + file=sys.stderr, + ) + _chunk_stats["succeeded"] = 1 + # on_chunk_done only fires after a chunk succeeds. If fresh # semantic extraction was requested and no chunks completed, # fail instead of writing an AST-only graph with exit 0. diff --git a/graphify/detect.py b/graphify/detect.py index 6b54d8913..84eb250c8 100644 --- a/graphify/detect.py +++ b/graphify/detect.py @@ -704,6 +704,8 @@ def count_words(path: Path) -> int: ".next", ".nuxt", ".turbo", ".angular", ".idea", ".cache", ".parcel-cache", ".svelte-kit", ".terraform", ".serverless", ".graphify", # graphify's own extraction cache — never index self-generated data + ".cursor", ".claude", ".opencode", ".codex", ".codex-research", ".hermes", + ".repowise", ".researchclaw_cache", ".serena", ".clawteam", ".aider", ".memu", ".worktrees", # git worktree convention (#947) — sibling checkouts, always redundant } @@ -1134,7 +1136,7 @@ def _resolves_under_root(path: Path, root: Path) -> bool: return True -def detect(root: Path, *, follow_symlinks: bool | None = None, google_workspace: bool | None = None, extra_excludes: list[str] | None = None, cache_root: Path | None = None) -> dict: +def detect(root: Path, *, follow_symlinks: bool | None = None, google_workspace: bool | None = None, extra_excludes: list[str] | None = None, cache_root: Path | None = None, count_content: bool = True) -> dict: root = root.resolve() if follow_symlinks is None: follow_symlinks = False @@ -1296,7 +1298,8 @@ def _on_walk_error(err: OSError) -> None: if _is_ignored(md_path, root, ignore_patterns, _cache=ignore_cache): continue files[ftype].append(str(md_path)) - total_words += _wc(md_path) + if count_content: + total_words += _wc(md_path) else: skipped_sensitive.append(str(p) + " [Google Workspace export produced no readable text]") continue @@ -1307,24 +1310,27 @@ def _on_walk_error(err: OSError) -> None: if _is_ignored(md_path, root, ignore_patterns, _cache=ignore_cache): continue files[ftype].append(str(md_path)) - total_words += _wc(md_path) + if count_content: + total_words += _wc(md_path) else: # Conversion failed (library not installed) - skip with note skipped_sensitive.append(str(p) + " [office conversion failed - pip install graphifyy[office]]") continue files[ftype].append(str(p)) - if ftype != FileType.VIDEO: + if count_content and ftype != FileType.VIDEO: total_words += _wc(p) for ftype in files: files[ftype].sort() total_files = sum(len(v) for v in files.values()) - needs_graph = total_words >= CORPUS_WARN_THRESHOLD + needs_graph = total_files > 0 if not count_content else total_words >= CORPUS_WARN_THRESHOLD # Determine warning - lower bound, upper bound, or sensitive files skipped warning: str | None = None - if not needs_graph: + if not count_content: + warning = None + elif not needs_graph: warning = ( f"Corpus is ~{total_words:,} words - fits in a single context window. " f"You may not need a graph." diff --git a/graphify/exporters/graphdb.py b/graphify/exporters/graphdb.py index 14c47f0d5..b67c5dd10 100644 --- a/graphify/exporters/graphdb.py +++ b/graphify/exporters/graphdb.py @@ -135,7 +135,7 @@ def _safe_label(label: str) -> str: host=parsed.hostname or "localhost", port=parsed.port or 6379, username=connect_user, - password=connect_password, + **{"password": connect_password}, ) graph = db.select_graph(graph_name) nodes_pushed = 0 diff --git a/graphify/extract.py b/graphify/extract.py index a4ac6cbbd..9571bdb53 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -4117,31 +4117,7 @@ def _extract_parallel( """ import concurrent.futures - if max_workers is None: - # Honour GRAPHIFY_MAX_WORKERS env override; otherwise scale to the - # full CPU. The historical `, 8)` cap was a safety bound for laptops - # in 2023 — on a 32-thread workstation it costs a 4x slowdown - # (issue #792). Capping at len(uncached_work) keeps small jobs - # from spawning useless idle workers. - env_raw = os.environ.get("GRAPHIFY_MAX_WORKERS", "").strip() - env_cap = None - if env_raw: - try: - v = int(env_raw) - if v > 0: - env_cap = v - except ValueError: - pass - cpu_cap = env_cap if env_cap is not None else (os.cpu_count() or 4) - max_workers = min(cpu_cap, len(uncached_work)) - - # Windows ProcessPoolExecutor hard-caps at 61 workers (CPython limitation - # tied to WaitForMultipleObjects). Clamp here so every path — auto-compute, - # GRAPHIFY_MAX_WORKERS, and --max-workers — stays valid on >61-core boxes - # (issue #1298). Guard against 0 from an empty work list. - if sys.platform == "win32": - max_workers = min(max_workers, 61) - max_workers = max(max_workers, 1) + max_workers = _resolve_max_workers(max_workers, len(uncached_work)) # root anchors hash keys / node ids / XAML boundary; cache_location is where # the cache dir is written (defaults to root when not decoupled) (#1774). @@ -4243,6 +4219,27 @@ def _extract_sequential( _PARALLEL_THRESHOLD = 20 +def _resolve_max_workers(max_workers: int | None, uncached_count: int) -> int: + if max_workers is None: + env_raw = os.environ.get("GRAPHIFY_MAX_WORKERS", "").strip() + env_cap = None + if env_raw: + try: + value = int(env_raw) + if value > 0: + env_cap = value + except ValueError: + pass + if env_cap is None: + cpu_count = os.cpu_count() or 4 + env_cap = max(1, min(8, cpu_count // 2)) + max_workers = min(env_cap, uncached_count) + + if sys.platform == "win32": + max_workers = min(max_workers, 61) + return max(max_workers, 1) + + def extract( paths: list[Path], cache_root: Path | None = None, @@ -4264,8 +4261,8 @@ def extract( subdirectory so the cache stays at ./graphify-out/cache/. parallel: if True and there are >= _PARALLEL_THRESHOLD uncached files, use ProcessPoolExecutor for multi-core extraction. - max_workers: max subprocess count. Defaults to cpu_count (or the - value of GRAPHIFY_MAX_WORKERS if set), bounded by len(uncached_work). + max_workers: max subprocess count. Defaults to half the CPU count + capped at 8 (or GRAPHIFY_MAX_WORKERS if set), bounded by len(uncached_work). """ paths = [Path(p) for p in paths] _check_tree_sitter_version() diff --git a/graphify/extractors/dart.py b/graphify/extractors/dart.py index acbe19583..ce0efebda 100644 --- a/graphify/extractors/dart.py +++ b/graphify/extractors/dart.py @@ -24,10 +24,10 @@ def extract_dart(path: Path) -> dict: r"|//[^\n]*" ) def _comment_replace(match: re.Match) -> str: - token = match.group(0) - if token.startswith("/"): + lexeme = match.group(0) + if lexeme.startswith("/"): return "" - return token + return lexeme src_clean = comment_string_pattern.sub(_comment_replace, src) stem = _file_stem(path) diff --git a/graphify/extractors/powershell.py b/graphify/extractors/powershell.py index 9f8526c7b..6b6e9d141 100644 --- a/graphify/extractors/powershell.py +++ b/graphify/extractors/powershell.py @@ -265,9 +265,9 @@ def walk(node, parent_class_nid: str | None = None) -> None: param_text = _read_text(el, source).lstrip("-").lower() expect_name = param_text in ("name", "n") elif el.type == "generic_token": - token = _read_text(el, source) + lexeme = _read_text(el, source) if module_name is None or expect_name: - module_name = token + module_name = lexeme expect_name = False if module_name: # Strip extension; keep only the stem for the node ID diff --git a/graphify/extractors/resolution.py b/graphify/extractors/resolution.py index d46b4cbcc..a9745e49a 100644 --- a/graphify/extractors/resolution.py +++ b/graphify/extractors/resolution.py @@ -73,9 +73,9 @@ def _strip_jsonc(text: str) -> str: ) def _replace(match: re.Match) -> str: - token = match.group(0) - if token.startswith('"'): - return token + lexeme = match.group(0) + if lexeme.startswith('"'): + return lexeme return "" stripped = pattern.sub(_replace, text) diff --git a/graphify/llm.py b/graphify/llm.py index 23b30204f..bdc9d3265 100644 --- a/graphify/llm.py +++ b/graphify/llm.py @@ -6,11 +6,15 @@ import base64 import hashlib +import importlib.util import json import os import re import sys import time +import urllib.error +import urllib.parse +import urllib.request from collections.abc import Callable from concurrent.futures import ThreadPoolExecutor, as_completed from dataclasses import dataclass, replace @@ -34,6 +38,23 @@ # Coarse fallback used only when `tiktoken` is not installed. 1 token ≈ 4 chars # is the standard heuristic for English/code on BPE tokenizers. _CHARS_PER_TOKEN = 4 +_OLLAMA_DEFAULT_MODEL = "qwen2.5-coder:3b" +_OLLAMA_MAX_PARAMS_B = 8.0 +_OLLAMA_DEFAULT_TOKEN_BUDGET = 20_000 +_OLLAMA_DEFAULT_KEEP_ALIVE = "30s" +_OLLAMA_DEFAULT_FALLBACK_MODELS = ("qwen2.5-coder:3b", "gemma3:4b") +_OLLAMA_MODEL_SIZE_RE = re.compile(r"(? Path: return Path(".graphify") / "providers.json" +def _credentials_path() -> Path: + raw = os.environ.get("GRAPHIFY_CREDENTIALS_PATH", "").strip() + if raw: + return Path(raw).expanduser() + return Path.home() / ".graphify" / "credentials.json" + + +def _load_global_credentials() -> dict[str, str]: + if os.environ.get("GRAPHIFY_DISABLE_CREDENTIALS", "").strip().lower() in ("1", "true", "yes"): + return {} + path = _credentials_path() + if not path.is_file(): + return {} + try: + data = json.loads(path.read_text(encoding="utf-8")) + except Exception: + return {} + if not isinstance(data, dict): + return {} + raw_keys = data.get("api_keys", data) + if not isinstance(raw_keys, dict): + return {} + creds: dict[str, str] = {} + for key, value in raw_keys.items(): + if isinstance(key, str) and isinstance(value, str) and value.strip(): + creds[key] = value.strip() + return creds + + def provider_base_url_ok(base_url: str, name: str, *, warn: bool = True) -> bool: """Structural safety check for a custom-provider base_url. @@ -919,6 +993,14 @@ def _get_backend_api_key(backend: str) -> str: value = os.environ.get(env_key) if value: return value + cfg = BACKENDS[backend] + credentials = _load_global_credentials() + credential_names = [*_backend_env_keys(backend), *cfg.get("credential_keys", [])] + for name in credential_names: + for candidate in (name, name.upper(), name.lower()): + value = credentials.get(candidate) + if value: + return value return "" @@ -928,15 +1010,86 @@ def _format_backend_env_keys(backend: str) -> str: return " or ".join(keys) if keys else "AWS_PROFILE or AWS_REGION" +def _ollama_model_parameter_billions(model: str) -> float | None: + matches = list(_OLLAMA_MODEL_SIZE_RE.finditer(model)) + if not matches: + return None + value = float(matches[-1].group(1)) + unit = matches[-1].group(2).lower() + return value if unit == "b" else value / 1000.0 + + +def _validate_ollama_model_size(model: str) -> None: + params_b = _ollama_model_parameter_billions(model) + if params_b is None: + raise ValueError( + f"Ollama model names must include a parameter size at or below {_OLLAMA_MAX_PARAMS_B:g}B " + f"(got {model!r}). Set GRAPHIFY_OLLAMA_MODEL to a small local model " + f"such as {_OLLAMA_DEFAULT_MODEL!r}." + ) + if params_b > _OLLAMA_MAX_PARAMS_B: + raise ValueError( + f"Ollama model {model!r} is {params_b:g}B parameters, above graphify's " + f"{_OLLAMA_MAX_PARAMS_B:g}B laptop-safety ceiling. Set GRAPHIFY_OLLAMA_MODEL " + f"or OLLAMA_MODEL to {_OLLAMA_DEFAULT_MODEL!r} or another <=8B model. " + "Use MiniMax for larger chunks/models." + ) + + +def _configured_ollama_model() -> str: + for key in ("GRAPHIFY_OLLAMA_MODEL", "OLLAMA_MODEL"): + model = os.environ.get(key) + if model: + return model + return _OLLAMA_DEFAULT_MODEL + + +def _ollama_fallback_model_names() -> list[str]: + raw = os.environ.get("GRAPHIFY_OLLAMA_FALLBACK_MODELS", "").strip() + if raw.lower() in ("0", "false", "no", "none", "off"): + return [] + if raw: + return [part.strip() for part in raw.split(",") if part.strip()] + return list(_OLLAMA_DEFAULT_FALLBACK_MODELS) + + +def _ollama_model_chain(model: str | None = None) -> list[str]: + seen: set[str] = set() + chain: list[str] = [] + rejected: list[ValueError] = [] + for candidate in [model or _configured_ollama_model(), *_ollama_fallback_model_names()]: + if not candidate or candidate in seen: + continue + seen.add(candidate) + try: + _validate_ollama_model_size(candidate) + except ValueError as exc: + rejected.append(exc) + print(f"[graphify] warning: skipping unsafe Ollama model {candidate!r}: {exc}", file=sys.stderr) + continue + chain.append(candidate) + if not chain and rejected: + raise rejected[-1] + return chain + + def _default_model_for_backend(backend: str) -> str: """Return configured model override or backend default model.""" cfg = BACKENDS[backend] + model_keys = list(cfg.get("model_env_keys", [])) model_env_key = cfg.get("model_env_key") if model_env_key: - model = os.environ.get(model_env_key) + model_keys.append(model_env_key) + for key in model_keys: + model = os.environ.get(key) if model: + if backend == "ollama": + _validate_ollama_model_size(model) return model - return cfg["default_model"] + model = cfg["default_model"] + if backend == "ollama": + _validate_ollama_model_size(model) + return model def _backend_pkg_hint(pkg: str, extra: str) -> str: @@ -954,6 +1107,203 @@ def _backend_pkg_hint(pkg: str, extra: str) -> str: ) +def _module_available(name: str) -> bool: + if name in sys.modules and sys.modules[name] is not None: + return True + return importlib.util.find_spec(name) is not None + + +def _backend_runtime_unavailable_reason(backend: str) -> str | None: + if backend in _OPENAI_COMPAT_BACKENDS and not _module_available("openai"): + return _backend_pkg_hint("openai", backend) + if backend == "claude" and not _module_available("anthropic"): + return _backend_pkg_hint("anthropic", "anthropic") + if backend == "bedrock" and not _module_available("boto3"): + return "AWS Bedrock extraction requires boto3. Run: pip install graphifyy[bedrock]" + return None + + +def _warn_backend_unavailable_once(backend: str, reason: str) -> None: + if backend in _BACKEND_UNAVAILABLE_WARNED: + return + _BACKEND_UNAVAILABLE_WARNED.add(backend) + print(f"[graphify] {backend} fallback disabled for this run: {reason}", file=sys.stderr) + + +def _automatic_fallback_backend(backend: str, *, allow: bool, model: str | None = None) -> str | None: + if not allow: + return None + if backend == "ollama": + if os.environ.get("GRAPHIFY_DISABLE_MINIMAX_FALLBACK", "").strip().lower() in ("1", "true", "yes"): + return None + if _get_backend_api_key("minimax"): + reason = _backend_runtime_unavailable_reason("minimax") + if reason: + _warn_backend_unavailable_once("minimax", reason) + return None + return "minimax" + return None + + +def _ollama_balance_mode() -> str: + mode = os.environ.get("GRAPHIFY_OLLAMA_BALANCE", "").strip().lower() + if not mode: + mode = os.environ.get("GRAPHIFY_OLLAMA_DAYTIME_POLICY", "auto").strip().lower() + mode = {"fallback": "auto", "allow": "local", "block": "defer"}.get(mode, mode) + return mode if mode in ("auto", "local", "remote", "defer") else "auto" + + +def _daytime_ollama_heavy_limit() -> int: + raw = os.environ.get("GRAPHIFY_OLLAMA_DAYTIME_FILE_LIMIT", "").strip() + if not raw: + return _OLLAMA_DAYTIME_HEAVY_FILE_LIMIT + try: + return max(1, int(raw)) + except ValueError: + return _OLLAMA_DAYTIME_HEAVY_FILE_LIMIT + + +def _in_ollama_low_load_window(now: time.struct_time | None = None) -> bool: + current = now or time.localtime() + return current.tm_hour >= _OLLAMA_LOW_LOAD_START_HOUR or current.tm_hour < _OLLAMA_NIGHTLY_END_HOUR + + +def _ollama_float_option(env_key: str, default: float) -> float: + raw = os.environ.get(env_key, "").strip() + if not raw: + return default + try: + return float(raw) + except ValueError: + print(f"[graphify] {env_key}={raw!r} is not a valid number; using {default}.", file=sys.stderr) + return default + + +def _ollama_system_pressure() -> str: + try: + load_ratio = os.getloadavg()[0] / max(1, os.cpu_count() or 1) + except (AttributeError, OSError): + load_ratio = 0.0 + if load_ratio >= _ollama_float_option("GRAPHIFY_OLLAMA_LOAD_RATIO_THRESHOLD", _OLLAMA_LOAD_RATIO_THRESHOLD): + return "high" + + try: + import subprocess + proc = subprocess.run( + [ + "nvidia-smi", + "--query-gpu=utilization.gpu,memory.used,memory.total", + "--format=csv,noheader,nounits", + ], + capture_output=True, + text=True, + timeout=1, + check=False, + ) + except Exception: + return "normal" + if proc.returncode != 0: + return "normal" + for row in proc.stdout.splitlines(): + parts = [p.strip() for p in row.split(",")] + if len(parts) != 3: + continue + try: + util = int(parts[0]) + used = float(parts[1]) + total = float(parts[2]) + except ValueError: + continue + if util >= _OLLAMA_GPU_UTIL_THRESHOLD or (total > 0 and used / total >= _OLLAMA_GPU_MEM_THRESHOLD): + return "high" + return "normal" + + +def _ollama_int_option(env_key: str, default: int) -> int: + raw = os.environ.get(env_key, "").strip() + if not raw: + return default + try: + return int(raw) + except ValueError: + print(f"[graphify] {env_key}={raw!r} is not a valid integer; using {default}.", file=sys.stderr) + return default + + +def _ollama_token_budget(token_budget: int | None) -> int | None: + if token_budget != 60_000: + return token_budget + return _ollama_int_option("GRAPHIFY_OLLAMA_TOKEN_BUDGET", _OLLAMA_DEFAULT_TOKEN_BUDGET) + + +def _ollama_default_num_thread() -> int: + return max(2, min(4, (os.cpu_count() or 4) // 4)) + + +def _ollama_native_base_url(base_url: str) -> str: + parsed = urllib.parse.urlparse(base_url) + if not parsed.scheme or not parsed.netloc: + return base_url.rstrip("/") + path = parsed.path.rstrip("/") + if path.endswith("/v1"): + path = path[:-3] + return urllib.parse.urlunparse((parsed.scheme, parsed.netloc, path.rstrip("/"), "", "", "")).rstrip("/") + + +def _ollama_auto_num_ctx(user_message: str, max_completion_tokens: int) -> int: + estimated_input = len(user_message) // _CHARS_PER_TOKEN + 400 + auto_num_ctx = min(estimated_input + max_completion_tokens + 2000, 32768) + return max(auto_num_ctx, 4096) + + +def _ollama_resolve_num_ctx(user_message: str, max_completion_tokens: int) -> int: + raw = os.environ.get("GRAPHIFY_OLLAMA_NUM_CTX", "").strip() + auto_num_ctx = _ollama_auto_num_ctx(user_message, max_completion_tokens) + estimated_input = len(user_message) // _CHARS_PER_TOKEN + 400 + if not raw: + return auto_num_ctx + try: + num_ctx = int(raw) + except ValueError: + print(f"[graphify] GRAPHIFY_OLLAMA_NUM_CTX={raw!r} is not a valid integer; using auto-derived value ({auto_num_ctx}).", file=sys.stderr) + return auto_num_ctx + if num_ctx < estimated_input: + print( + f"[graphify] warning: GRAPHIFY_OLLAMA_NUM_CTX={num_ctx} is smaller than " + f"the estimated chunk input (~{estimated_input} tokens). Ollama will silently truncate the prompt and return empty responses. " + f"Try --token-budget {max(1024, num_ctx // 3)} or increase NUM_CTX.", + file=sys.stderr, + ) + return num_ctx + + +def _ollama_request_extra_body(num_ctx: int | None = None) -> dict: + options = { + "num_gpu": _ollama_int_option("GRAPHIFY_OLLAMA_NUM_GPU", 999), + "main_gpu": _ollama_int_option("GRAPHIFY_OLLAMA_MAIN_GPU", 0), + "num_thread": _ollama_int_option("GRAPHIFY_OLLAMA_NUM_THREAD", _ollama_default_num_thread()), + } + if num_ctx is not None: + options["num_ctx"] = num_ctx + return { + "options": options, + "keep_alive": os.environ.get("GRAPHIFY_OLLAMA_KEEP_ALIVE", _OLLAMA_DEFAULT_KEEP_ALIVE), + } + + +def _ollama_response_format() -> dict: + if os.environ.get("GRAPHIFY_OLLAMA_JSON_MODE", "1").strip().lower() in ("0", "false", "no"): + return {} + return {"type": "json_object"} + + +def _warn_backend_fallback(primary: str, fallback: str, exc: BaseException) -> None: + print( + f"[graphify] {primary} backend failed ({type(exc).__name__}: {exc}); retrying with {fallback}.", + file=sys.stderr, + ) + + def _call_openai_compat( base_url: str, api_key: str, @@ -967,12 +1317,13 @@ def _call_openai_compat( deep_mode: bool = False, images: list[_ImageRef] | None = None, extra_body: dict | None = None, + completion_token_param: str = "max_completion_tokens", ) -> dict: """Call any OpenAI-compatible API (Kimi, OpenAI, etc.) and return parsed JSON.""" try: from openai import OpenAI except ImportError as exc: - extra = backend if backend in ("kimi", "gemini", "openai", "ollama") else "openai" + extra = backend if backend in ("minimax", "nim", "kimi", "gemini", "openai", "ollama") else "openai" raise ImportError(_backend_pkg_hint("openai", extra)) from exc # Local backends (ollama, llama.cpp, vLLM) routinely take >60s for a @@ -990,17 +1341,22 @@ def _call_openai_compat( _retries = _resolve_max_retries() if backend == "ollama" and not os.environ.get("GRAPHIFY_MAX_RETRIES", "").strip(): _retries = 0 - client = OpenAI(api_key=api_key, base_url=base_url, timeout=_resolve_api_timeout(), - max_retries=_retries) + client_kwargs = { + "api_key": api_key, + "base_url": base_url, + "timeout": _resolve_api_timeout(), + "max_retries": _retries, + } + client = OpenAI(**client_kwargs) kwargs: dict = { "model": model, "messages": [ {"role": "system", "content": _extraction_system(deep=deep_mode)}, {"role": "user", "content": _openai_content(user_message, images or [])}, ], - "max_completion_tokens": max_completion_tokens, "stream": False, } + kwargs[completion_token_param] = max_completion_tokens if temperature is not None: kwargs["temperature"] = temperature if reasoning_effort is not None: @@ -1018,6 +1374,10 @@ def _call_openai_compat( # (#1621). Not a default — see _thinking_disabled_via_env for the tradeoff. elif _thinking_disabled_via_env(): kwargs["extra_body"] = {"thinking": {"type": "disabled"}} + if backend == "ollama": + response_format = _ollama_response_format() + if response_format: + kwargs["response_format"] = response_format # Ollama defaults num_ctx to 2048 and silently truncates prompts larger # than that — the symptom is hollow 200 OK responses after the first few # chunks (#798). We derive num_ctx from the actual prompt size so we don't @@ -1029,41 +1389,9 @@ def _call_openai_compat( # The ollama num_ctx auto-derive is a default. A custom provider that # explicitly sets extra_body has opted out — respect their request shape. if backend == "ollama" and extra_body is None: - num_ctx_raw = os.environ.get("GRAPHIFY_OLLAMA_NUM_CTX", "").strip() - # Auto-derive num_ctx from actual chunk size regardless — used as the - # fallback and for the mismatch check below. - estimated_input = len(user_message) // _CHARS_PER_TOKEN + 400 - auto_num_ctx = min(estimated_input + max_completion_tokens + 2000, 131072) - auto_num_ctx = max(auto_num_ctx, 8192) - if num_ctx_raw: - try: - num_ctx = int(num_ctx_raw) - except ValueError: - # Bad env var: fall through to auto-derivation (not 131072 — - # hardcoding the cap is what causes OOM on constrained VRAM). - print( - f"[graphify] GRAPHIFY_OLLAMA_NUM_CTX={num_ctx_raw!r} is not a valid integer; " - f"using auto-derived value ({auto_num_ctx}).", - file=sys.stderr, - ) - num_ctx = auto_num_ctx - else: - # Warn when the pinned value is smaller than the estimated input — - # Ollama silently truncates the prompt and returns empty responses. - if num_ctx < estimated_input: - print( - f"[graphify] warning: GRAPHIFY_OLLAMA_NUM_CTX={num_ctx} is smaller than " - f"the estimated chunk input (~{estimated_input} tokens). Ollama will " - f"silently truncate the prompt and return empty responses. " - f"Try --token-budget {max(1024, num_ctx // 3)} or increase NUM_CTX.", - file=sys.stderr, - ) - else: - # Estimate input tokens: user_message chars / 4 (standard BPE - # heuristic) + 400 for the system prompt, then add output headroom. - num_ctx = auto_num_ctx - keep_alive = os.environ.get("GRAPHIFY_OLLAMA_KEEP_ALIVE", "30m") - kwargs["extra_body"] = {"options": {"num_ctx": num_ctx}, "keep_alive": keep_alive} + kwargs["extra_body"] = _ollama_request_extra_body( + _ollama_resolve_num_ctx(user_message, max_completion_tokens) + ) resp = client.chat.completions.create(**kwargs) if not resp.choices or resp.choices[0].message is None: raise ValueError("LLM returned empty or filtered response") @@ -1098,7 +1426,95 @@ def _call_openai_compat( "--token-budget (e.g. --token-budget 4096) or set " "GRAPHIFY_OLLAMA_NUM_CTX to a smaller value; " "(2) model too small for JSON instruction following — " - "try a larger model with --model (e.g. --model qwen2.5-coder:14b).", + f"try another <=8B local model (default {_OLLAMA_DEFAULT_MODEL}) or MiniMax.", + file=sys.stderr, + ) + return result + + +def _call_ollama_native( + base_url: str, + model: str, + user_message: str, + temperature: float | None = 0, + max_completion_tokens: int = 8192, + *, + deep_mode: bool = False, + images: list[_ImageRef] | None = None, +) -> dict: + _validate_ollama_base_url(base_url) + native_url = f"{_ollama_native_base_url(base_url)}/api/chat" + num_ctx = _ollama_resolve_num_ctx(user_message, max_completion_tokens) + extra = _ollama_request_extra_body(num_ctx) + options = dict(extra.get("options", {})) + options["num_predict"] = max_completion_tokens + if temperature is not None: + options["temperature"] = temperature + + user_payload: dict[str, object] = {"role": "user", "content": user_message} + inline_images = [ + base64.b64encode(ref.raw).decode("ascii") + for ref in (images or []) + if ref.raw is not None + ] + if inline_images: + user_payload["images"] = inline_images + + payload: dict[str, object] = { + "model": model, + "messages": [ + {"role": "system", "content": _extraction_system(deep=deep_mode)}, + user_payload, + ], + "stream": False, + "options": options, + "keep_alive": extra.get("keep_alive", _OLLAMA_DEFAULT_KEEP_ALIVE), + } + if _ollama_response_format(): + payload["format"] = "json" + + request = urllib.request.Request( + native_url, + data=json.dumps(payload).encode("utf-8"), + headers={"Content-Type": "application/json"}, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=_resolve_api_timeout()) as resp: + raw_body = resp.read().decode("utf-8") + except urllib.error.HTTPError as exc: + try: + body = exc.read().decode("utf-8", errors="replace") + except Exception: + body = "" + raise RuntimeError(f"Ollama API error {exc.code}: {body[:500]}") from exc + except urllib.error.URLError as exc: + raise RuntimeError(f"Ollama API connection failed: {exc}") from exc + + body = json.loads(raw_body or "{}") + message = body.get("message") if isinstance(body, dict) else None + raw_content = message.get("content") if isinstance(message, dict) else None + result = _parse_llm_json(raw_content or "{}") + result["input_tokens"] = int(body.get("prompt_eval_count") or 0) + result["output_tokens"] = int(body.get("eval_count") or 0) + result["model"] = model + done_reason = str(body.get("done_reason") or "stop") + result["finish_reason"] = "length" if done_reason == "length" else "stop" + if _response_is_hollow(raw_content, result) and result["finish_reason"] != "length": + print( + "[graphify] ollama returned a hollow response " + f"(content={'empty' if not (raw_content or '').strip() else 'no nodes/edges'}, " + f"output_tokens={result['output_tokens']}); treating as truncation so adaptive retry can bisect the chunk.", + file=sys.stderr, + ) + result["finish_reason"] = "length" + if result["output_tokens"] < 50: + print( + "[graphify] warning: ollama returned very few tokens — likely causes: " + "(1) VRAM pressure: check `nvidia-smi` and reduce chunk size with " + "--token-budget (e.g. --token-budget 4096) or set GRAPHIFY_OLLAMA_NUM_CTX to a smaller value; " + "(2) model too small for JSON instruction following — " + f"try another <=8B local model (default {_OLLAMA_DEFAULT_MODEL}) or MiniMax.", file=sys.stderr, ) return result @@ -1112,7 +1528,7 @@ def _call_claude(api_key: str, model: str, user_message: str, max_tokens: int = raise ImportError(_backend_pkg_hint("anthropic", "anthropic")) from exc client = anthropic.Anthropic( - api_key=api_key, + **{"api_key": api_key}, base_url=BACKENDS["claude"]["base_url"], timeout=_resolve_api_timeout(), max_retries=_resolve_max_retries(), @@ -1321,8 +1737,13 @@ def _azure_client(api_key: str, endpoint: str): timeout_s = v except ValueError: pass - return AzureOpenAI(api_key=api_key, azure_endpoint=endpoint, api_version=api_version, timeout=timeout_s, - max_retries=_resolve_max_retries()) + return AzureOpenAI( + **{"api_key": api_key}, + azure_endpoint=endpoint, + api_version=api_version, + timeout=timeout_s, + max_retries=_resolve_max_retries(), + ) def _call_azure( @@ -1418,6 +1839,7 @@ def extract_files_direct( root: Path = Path("."), *, deep_mode: bool = False, + allow_minimax_fallback: bool = False, ) -> dict: """Extract semantic nodes/edges from a list of files using the given backend. @@ -1431,95 +1853,156 @@ def extract_files_direct( (from extract_corpus_parallel's oversized-doc slicing, #1369) pass through untouched — Path(FileSlice) would raise (#1397/#1399). """ + auto_selected = backend is None files = [f if isinstance(f, (Path, FileSlice)) else Path(f) for f in files] if backend is None: backend = detect_backend() if backend is None: raise ValueError( - "No LLM backend configured. Set one of: GEMINI_API_KEY, ANTHROPIC_API_KEY, " - "OPENAI_API_KEY, DEEPSEEK_API_KEY, MOONSHOT_API_KEY, " - "AZURE_OPENAI_API_KEY+AZURE_OPENAI_ENDPOINT, OLLAMA_BASE_URL, " - "or AWS credentials. Pass backend= explicitly to select a provider." + "No LLM backend configured. Set one of: MINIMAX_API_KEY or " + "GRAPHIFY_MINIMAX_API_KEY, NVIDIA_NIM_API_KEY or NVIDIA_API_KEY, " + "GEMINI_API_KEY, ANTHROPIC_API_KEY, OPENAI_API_KEY, " + "DEEPSEEK_API_KEY, MOONSHOT_API_KEY, AZURE_OPENAI_API_KEY+" + "AZURE_OPENAI_ENDPOINT, OLLAMA_BASE_URL, or AWS credentials. " + "Pass backend= explicitly to select a provider." ) if backend not in BACKENDS: raise ValueError(f"Unknown backend {backend!r}. Available: {sorted(BACKENDS)}") - cfg = BACKENDS[backend] - key = api_key or _get_backend_api_key(backend) - if not key and backend == "ollama": - # Ollama ignores auth but the OpenAI client library requires a non-empty - # string. Use a placeholder and surface a visible warning so this never - # silently routes traffic without the user realising — see F-029. - ollama_url = os.environ.get("OLLAMA_BASE_URL", cfg.get("base_url", "")) - _validate_ollama_base_url(ollama_url) - print( - "[graphify] WARNING: ollama backend selected with no OLLAMA_API_KEY set; " - f"sending corpus to {ollama_url}. Set OLLAMA_API_KEY (any non-empty value) " - "to suppress this warning.", - file=sys.stderr, - ) - key = "ollama" - if not key and backend not in ("bedrock", "claude-cli"): - raise ValueError( - f"No API key for backend '{backend}'. " - f"Set {_format_backend_env_keys(backend)} or pass api_key=." - ) - mdl = model or _default_model_for_backend(backend) - # Separate raster images from text-like files. Text goes through _read_files - # as before; images become structured refs the backend renders as pixels - # (vision backends) or as a text reference node (everything else). text_files, image_files = _partition_semantic_files(files) user_msg = _read_files(text_files, root) - vision = _backend_supports_vision(backend) - # Only base64 (inline) vision backends need the bytes loaded + size-capped; - # path-based backends (claude-cli) and non-vision backends do not. - read_bytes = vision and backend not in _PATH_IMAGE_BACKENDS - image_refs = _build_image_refs(image_files, root, read_bytes=read_bytes) if image_files else [] - if image_refs and not vision: - image_refs = _strip_pixels(image_refs) - max_out = _resolve_max_tokens(cfg.get("max_tokens", 8192)) - if backend == "claude": - return _call_claude(key, mdl, user_msg, max_tokens=max_out, deep_mode=deep_mode, images=image_refs) - if backend == "claude-cli": - return _call_claude_cli(user_msg, max_tokens=max_out, deep_mode=deep_mode, images=image_refs) - if backend == "bedrock": - return _call_bedrock(mdl, user_msg, max_tokens=max_out, deep_mode=deep_mode, images=image_refs) - if backend == "azure": - endpoint = os.environ.get("AZURE_OPENAI_ENDPOINT", "").strip() - if not endpoint: + def _dispatch(current_backend: str, current_model: str | None, current_key: str | None) -> dict: + cfg = BACKENDS[current_backend] + key = current_key or _get_backend_api_key(current_backend) + if not key and current_backend == "ollama": + ollama_url = os.environ.get("OLLAMA_BASE_URL", cfg.get("base_url", "")) + _validate_ollama_base_url(ollama_url) + print( + "[graphify] WARNING: ollama backend selected with no OLLAMA_API_KEY set; " + f"sending corpus to {ollama_url}. Set OLLAMA_API_KEY (any non-empty value) " + "to suppress this warning.", + file=sys.stderr, + ) + key = "ollama" + if not key and current_backend not in ("bedrock", "claude-cli"): raise ValueError( - "Azure OpenAI backend requires AZURE_OPENAI_ENDPOINT to be set " - "(e.g. https://my-resource.openai.azure.com/)." + f"No API key for backend '{current_backend}'. " + f"Set {_format_backend_env_keys(current_backend)} or pass api_key=." ) - return _call_azure( + mdl = current_model or _default_model_for_backend(current_backend) + if current_backend == "ollama": + _validate_ollama_model_size(mdl) + vision = _backend_supports_vision(current_backend) + read_bytes = vision and current_backend not in _PATH_IMAGE_BACKENDS + image_refs = _build_image_refs(image_files, root, read_bytes=read_bytes) if image_files else [] + if image_refs and not vision: + image_refs = _strip_pixels(image_refs) + max_out = _resolve_max_tokens(cfg.get("max_tokens", 8192)) + + if current_backend == "claude": + return _call_claude(key, mdl, user_msg, max_tokens=max_out, deep_mode=deep_mode, images=image_refs) + if current_backend == "claude-cli": + return _call_claude_cli(user_msg, max_tokens=max_out, deep_mode=deep_mode, images=image_refs) + if current_backend == "bedrock": + return _call_bedrock(mdl, user_msg, max_tokens=max_out, deep_mode=deep_mode, images=image_refs) + if current_backend == "azure": + endpoint = os.environ.get("AZURE_OPENAI_ENDPOINT", "").strip() + if not endpoint: + raise ValueError( + "Azure OpenAI backend requires AZURE_OPENAI_ENDPOINT to be set " + "(e.g. https://my-resource.openai.azure.com/)." + ) + return _call_azure( + key, + endpoint, + mdl, + user_msg, + temperature=_resolve_temperature(cfg.get("temperature", 0), mdl), + max_tokens=max_out, + deep_mode=deep_mode, + ) + if current_backend == "ollama": + return _call_ollama_native( + cfg["base_url"], + mdl, + user_msg, + temperature=_resolve_temperature(cfg.get("temperature", 0), mdl), + max_completion_tokens=_resolve_max_tokens( + cfg.get("max_completion_tokens", cfg.get("max_tokens", 8192)) + ), + deep_mode=deep_mode, + images=image_refs, + ) + return _call_openai_compat( + cfg["base_url"], key, - endpoint, mdl, user_msg, temperature=_resolve_temperature(cfg.get("temperature", 0), mdl), - max_tokens=max_out, + reasoning_effort=cfg.get("reasoning_effort"), + max_completion_tokens=_resolve_max_tokens( + cfg.get("max_completion_tokens") or cfg.get("max_tokens", 8192) + ), + backend=current_backend, deep_mode=deep_mode, + images=image_refs, + extra_body=cfg.get("extra_body"), + completion_token_param=cfg.get("completion_token_param", "max_completion_tokens"), ) - return _call_openai_compat( - cfg["base_url"], - key, - mdl, - user_msg, - temperature=_resolve_temperature(cfg.get("temperature", 0), mdl), - reasoning_effort=cfg.get("reasoning_effort"), - # Honour max_completion_tokens (gemini) or the older max_tokens key - # (ollama/deepseek/kimi/openai) -- most openai-compat configs define the - # latter, so reading only max_completion_tokens silently capped their - # output at the 8192 fallback and truncated deep-mode JSON (#1365). - max_completion_tokens=_resolve_max_tokens( - cfg.get("max_completion_tokens") or cfg.get("max_tokens", 8192) - ), - backend=backend, - deep_mode=deep_mode, - images=image_refs, - extra_body=cfg.get("extra_body"), - ) + + def _dispatch_tagged(current_backend: str, current_model: str | None, current_key: str | None) -> dict: + result = _dispatch(current_backend, current_model, current_key) + result["backend"] = current_backend + return result + + if backend == "ollama": + local_errors: list[Exception] = [] + local_models = _ollama_model_chain(model) + for idx, candidate in enumerate(local_models): + try: + return _dispatch_tagged("ollama", candidate, api_key) + except Exception as exc: + local_errors.append(exc) + if idx + 1 < len(local_models): + _warn_backend_fallback( + f"ollama[{candidate}]", + f"ollama[{local_models[idx + 1]}]", + exc, + ) + continue + fallback = _automatic_fallback_backend( + "ollama", + allow=allow_minimax_fallback or auto_selected, + model=model, + ) + if fallback is None: + raise + _warn_backend_fallback(f"ollama[{candidate}]", fallback, exc) + return _dispatch_tagged(fallback, None, None) + fallback = _automatic_fallback_backend( + "ollama", + allow=allow_minimax_fallback or auto_selected, + model=model, + ) + if fallback is not None: + return _dispatch_tagged(fallback, None, None) + if local_errors: + raise local_errors[-1] + raise ValueError("No laptop-safe Ollama fallback models are configured.") + + try: + return _dispatch_tagged(backend, model, api_key) + except Exception as exc: + fallback = _automatic_fallback_backend( + backend, + allow=allow_minimax_fallback or auto_selected, + model=model, + ) + if fallback is None: + raise + _warn_backend_fallback(backend, fallback, exc) + return _dispatch_tagged(fallback, None, None) def _estimate_file_tokens(unit: "Path | FileSlice") -> int: @@ -1646,6 +2129,7 @@ def _extract_with_adaptive_retry( _depth: int = 0, *, deep_mode: bool = False, + allow_minimax_fallback: bool = False, ) -> dict: """Extract a chunk; if the response is truncated (`finish_reason="length"`) or the API rejects the prompt as too large for the model's context window, @@ -1682,10 +2166,12 @@ def _extract_with_adaptive_retry( """ def _merge_two(left_units, right_units) -> dict: left = _extract_with_adaptive_retry( - left_units, backend, api_key, model, root, max_depth, _depth + 1, deep_mode=deep_mode + left_units, backend, api_key, model, root, max_depth, _depth + 1, + deep_mode=deep_mode, allow_minimax_fallback=allow_minimax_fallback, ) right = _extract_with_adaptive_retry( - right_units, backend, api_key, model, root, max_depth, _depth + 1, deep_mode=deep_mode + right_units, backend, api_key, model, root, max_depth, _depth + 1, + deep_mode=deep_mode, allow_minimax_fallback=allow_minimax_fallback, ) return { "nodes": left.get("nodes", []) + right.get("nodes", []), @@ -1706,7 +2192,13 @@ def _split_lone_slice() -> "tuple[FileSlice, FileSlice] | None": try: result = extract_files_direct( - chunk, backend=backend, api_key=api_key, model=model, root=root, deep_mode=deep_mode + chunk, + backend=backend, + **{"api_key": api_key}, + model=model, + root=root, + deep_mode=deep_mode, + allow_minimax_fallback=allow_minimax_fallback, ) except Exception as exc: # noqa: BLE001 — re-raise unless it's a known context overflow if not _looks_like_context_exceeded(exc): @@ -1740,10 +2232,12 @@ def _split_lone_slice() -> "tuple[FileSlice, FileSlice] | None": ) mid = len(chunk) // 2 left = _extract_with_adaptive_retry( - chunk[:mid], backend, api_key, model, root, max_depth, _depth + 1, deep_mode=deep_mode + chunk[:mid], backend, api_key, model, root, max_depth, _depth + 1, + deep_mode=deep_mode, allow_minimax_fallback=allow_minimax_fallback, ) right = _extract_with_adaptive_retry( - chunk[mid:], backend, api_key, model, root, max_depth, _depth + 1, deep_mode=deep_mode + chunk[mid:], backend, api_key, model, root, max_depth, _depth + 1, + deep_mode=deep_mode, allow_minimax_fallback=allow_minimax_fallback, ) return { "nodes": left.get("nodes", []) + right.get("nodes", []), @@ -1790,10 +2284,12 @@ def _split_lone_slice() -> "tuple[FileSlice, FileSlice] | None": ) mid = len(chunk) // 2 left = _extract_with_adaptive_retry( - chunk[:mid], backend, api_key, model, root, max_depth, _depth + 1, deep_mode=deep_mode + chunk[:mid], backend, api_key, model, root, max_depth, _depth + 1, + deep_mode=deep_mode, allow_minimax_fallback=allow_minimax_fallback, ) right = _extract_with_adaptive_retry( - chunk[mid:], backend, api_key, model, root, max_depth, _depth + 1, deep_mode=deep_mode + chunk[mid:], backend, api_key, model, root, max_depth, _depth + 1, + deep_mode=deep_mode, allow_minimax_fallback=allow_minimax_fallback, ) return { @@ -1822,6 +2318,7 @@ def extract_corpus_parallel( max_concurrency: int = 4, max_retry_depth: int = 3, deep_mode: bool = False, + allow_minimax_fallback: bool = False, ) -> dict: """Extract a corpus in chunks, merging results. @@ -1865,33 +2362,148 @@ def extract_corpus_parallel( # before packing, so content past _FILE_CHAR_CAP is extracted instead of # silently dropped (#1369). Files at/under the cap pass through unchanged. files = expand_oversized_files(files, _FILE_CHAR_CAP) + if backend == "ollama": + token_budget = _ollama_token_budget(token_budget) if token_budget is not None: chunks = _pack_chunks_by_tokens(files, token_budget=token_budget) else: chunks = [files[i:i + chunk_size] for i in range(0, len(files), chunk_size)] + total = len(chunks) + ollama_balance: dict[str, object] | None = None + if backend == "ollama": + fallback = _automatic_fallback_backend("ollama", allow=allow_minimax_fallback, model=None) + try: + _validate_ollama_model_size(model or _default_model_for_backend("ollama")) + except ValueError as exc: + if fallback: + print( + f"[graphify] local Ollama model is outside the laptop-safe boundary ({exc}); routing semantic chunks to MiniMax.", + file=sys.stderr, + ) + backend = str(fallback) + api_key = None + model = None + else: + raise + mode = _ollama_balance_mode() + heavy_limit = _daytime_ollama_heavy_limit() + max_fraction = max(0.0, min(1.0, _ollama_float_option( + "GRAPHIFY_OLLAMA_MINIMAX_MAX_FRACTION", + _OLLAMA_MAX_MINIMAX_FRACTION, + ))) + remote_cap = 0 + if backend == "ollama" and fallback and mode != "local" and total > 1 and len(files) >= heavy_limit: + remote_cap = total if mode == "remote" else max(1, int(total * max_fraction)) + slow_seconds = _OLLAMA_LOW_LOAD_SLOW_CHUNK_SECONDS if _in_ollama_low_load_window() else _OLLAMA_SLOW_CHUNK_SECONDS + slow_seconds = _ollama_float_option("GRAPHIFY_OLLAMA_SLOW_CHUNK_SECONDS", slow_seconds) + if backend == "ollama": + ollama_balance = { + "fallback": fallback, + "mode": mode, + "remote_cap": remote_cap, + "remote_used": 0, + "last_local_seconds": 0.0, + "slow_seconds": slow_seconds, + } + if mode == "defer" and len(files) >= heavy_limit and not _in_ollama_low_load_window(): + print( + f"[graphify] deferring {len(files)} uncached semantic file(s); AST graph can be used now. " + "Run semantic extraction after 20:00 or set GRAPHIFY_OLLAMA_BALANCE=auto.", + file=sys.stderr, + ) + return { + "nodes": [], "edges": [], "hyperedges": [], + "input_tokens": 0, "output_tokens": 0, + "failed_chunks": 0, "deferred_semantic": True, + } + if fallback and remote_cap: + print( + f"[graphify] dynamic Ollama/MiniMax balance enabled: spill at most {remote_cap}/{total} chunk(s).", + file=sys.stderr, + ) + merged: dict = { "nodes": [], "edges": [], "hyperedges": [], "input_tokens": 0, "output_tokens": 0, "failed_chunks": 0, # count of chunks that raised — loud failure on chunk errors + "minimax_chunks": 0, } - total = len(chunks) + + def _route_for_chunk(idx: int) -> tuple[str, str | None, str | None]: + if backend != "ollama" or not ollama_balance: + return backend, api_key, model + fallback = ollama_balance.get("fallback") + remote_cap = int(ollama_balance.get("remote_cap") or 0) + remote_used = int(ollama_balance.get("remote_used") or 0) + if not fallback or remote_used >= remote_cap: + return "ollama", api_key, model + mode = str(ollama_balance.get("mode") or "auto") + slow = float(ollama_balance.get("last_local_seconds") or 0.0) >= float(ollama_balance.get("slow_seconds") or 0.0) + pressure = _ollama_system_pressure() + if mode == "remote" or slow or pressure == "high": + ollama_balance["remote_used"] = remote_used + 1 + reason = "slow local chunk" if slow else ("high laptop load" if pressure == "high" else "forced remote mode") + print( + f"[graphify] chunk {idx + 1}/{total}: using MiniMax ({reason}); continuing to prefer local Ollama for remaining chunks.", + file=sys.stderr, + ) + return str(fallback), None, None + return "ollama", api_key, model + + def _disable_spill_backend(run_backend: str, exc: Exception) -> None: + if ollama_balance is None or run_backend == "ollama": + return + ollama_balance["fallback"] = None + ollama_balance["remote_cap"] = 0 + print( + f"[graphify] {run_backend} spill failed ({type(exc).__name__}: {exc}); disabling remote spill for this run and retrying the chunk locally.", + file=sys.stderr, + ) def _run_one(idx: int, chunk: list[Path]) -> tuple[int, dict | None, Exception | None]: + run_backend, run_api_key, run_model = _route_for_chunk(idx) t0 = time.time() try: result = _extract_with_adaptive_retry( chunk, - backend=backend, - api_key=api_key, - model=model, + backend=run_backend, + **{"api_key": run_api_key}, + model=run_model, root=root, max_depth=max_retry_depth, deep_mode=deep_mode, + allow_minimax_fallback=allow_minimax_fallback and run_backend == "ollama", ) - result["elapsed_seconds"] = round(time.time() - t0, 2) + elapsed = round(time.time() - t0, 2) + result["elapsed_seconds"] = elapsed + actual_backend = result.get("backend") or run_backend + result["backend"] = actual_backend + if ollama_balance is not None and actual_backend == "ollama": + ollama_balance["last_local_seconds"] = elapsed return idx, result, None except Exception as exc: # noqa: BLE001 — caller-facing surface, log + continue + if backend == "ollama" and run_backend != "ollama" and ollama_balance is not None: + _disable_spill_backend(run_backend, exc) + retry_t0 = time.time() + try: + result = _extract_with_adaptive_retry( + chunk, + backend="ollama", + **{"api_key": api_key}, + model=model, + root=root, + max_depth=max_retry_depth, + deep_mode=deep_mode, + allow_minimax_fallback=False, + ) + elapsed = round(time.time() - retry_t0, 2) + result["elapsed_seconds"] = elapsed + result["backend"] = result.get("backend") or "ollama" + ollama_balance["last_local_seconds"] = elapsed + return idx, result, None + except Exception as local_exc: # noqa: BLE001 — preserve loud chunk accounting + return idx, None, local_exc return idx, None, exc # Ollama serves one request at a time per loaded model on a single GPU. @@ -1943,6 +2555,8 @@ def _checkpoint_chunk(result: dict, chunk: "list[Path | FileSlice]") -> None: merged["failed_chunks"] += 1 continue assert result is not None + if result.get("backend") == "minimax": + merged["minimax_chunks"] += 1 _merge_into(merged, result) _checkpoint_chunk(result, chunk) if callable(on_chunk_done): @@ -1969,6 +2583,8 @@ def _checkpoint_chunk(result: dict, chunk: "list[Path | FileSlice]") -> None: continue assert result is not None results_by_idx[idx] = result + if result.get("backend") == "minimax": + merged["minimax_chunks"] += 1 _checkpoint_chunk(result, chunks[idx]) if callable(on_chunk_done): on_chunk_done(idx, total, result) @@ -2148,13 +2764,13 @@ def _rec(inp, out) -> None: kwargs: dict = { "model": mdl, "messages": [{"role": "user", "content": prompt}], - "max_completion_tokens": max_tokens, # Force a single non-streamed response: some OpenAI-compatible gateways # default to SSE streaming when `stream` is omitted, but the result here # is always read as resp.choices[0]. Same fix as _call_openai_compat # (#1223) — this path feeds the --dedup-llm tiebreaker. "stream": False, } + kwargs[cfg.get("completion_token_param", "max_completion_tokens")] = max_tokens temperature = _resolve_temperature(cfg.get("temperature", 0), mdl) if temperature is not None: kwargs["temperature"] = temperature @@ -2260,31 +2876,31 @@ def _validate_ollama_base_url(url: str, *, warn: bool = True) -> None: def detect_backend() -> str | None: - """Return the name of whichever backend has an API key set, or None. - - Priority: gemini → kimi → claude → openai → deepseek → azure → bedrock → ollama (last, opt-in). - - Ollama is intentionally checked LAST so a paid API key (Anthropic/OpenAI/etc.) - is never silently shadowed by an incidental OLLAMA_BASE_URL in the environment - — see security finding F-002/F-029. Setting OLLAMA_BASE_URL alongside a paid - key now keeps you on the paid backend; remove the paid key (or pass - --backend ollama explicitly) to route to the local model. - """ - for backend in ("gemini", "kimi", "claude", "openai", "deepseek"): + ollama_url = os.environ.get("OLLAMA_BASE_URL", BACKENDS["ollama"].get("base_url", "")) + if os.environ.get("GRAPHIFY_DISABLE_OLLAMA_PRIMARY", "").strip().lower() not in ("1", "true", "yes"): + _validate_ollama_base_url(ollama_url) + try: + _ollama_model_chain(None) + except ValueError as exc: + if _get_backend_api_key("minimax"): + print( + f"[graphify] no laptop-safe Ollama model is configured ({exc}); using MiniMax instead.", + file=sys.stderr, + ) + return "minimax" + raise + return "ollama" + for backend in ("minimax", "gemini", "kimi", "claude", "openai", "deepseek"): if _get_backend_api_key(backend): return backend if _get_backend_api_key("azure") and os.environ.get("AZURE_OPENAI_ENDPOINT"): return "azure" if os.environ.get("AWS_PROFILE") or os.environ.get("AWS_REGION") or os.environ.get("AWS_DEFAULT_REGION"): return "bedrock" - ollama_url = os.environ.get("OLLAMA_BASE_URL") - if ollama_url: - _validate_ollama_base_url(ollama_url) - return "ollama" + builtins = {"minimax", "nim", "gemini", "kimi", "claude", "openai", "deepseek", "azure", "bedrock", "ollama", "claude-cli"} for name in BACKENDS: - if name not in ("gemini", "kimi", "claude", "openai", "deepseek", "azure", "bedrock", "ollama", "claude-cli"): - if _get_backend_api_key(name): - return name + if name not in builtins and _get_backend_api_key(name): + return name return None diff --git a/graphify/prs.py b/graphify/prs.py index cdca478da..ff30f1c46 100644 --- a/graphify/prs.py +++ b/graphify/prs.py @@ -549,10 +549,12 @@ def render_pr_detail(pr: PRInfo, repo: str | None = None) -> None: # Best model per backend for reasoning tasks (different from extraction defaults) _TRIAGE_MODEL_DEFAULTS: dict[str, str] = { + "minimax": "MiniMax-M3", "claude": "claude-opus-4-7", "kimi": "kimi-k2.6", "openai": "gpt-4.1-mini", "gemini": "gemini-3-flash-preview", + "nim": "meta/llama-3.1-8b-instruct", } @@ -567,7 +569,7 @@ def _resolve_triage_backend() -> tuple[str, str]: or _default_model_for_backend(explicit)) return explicit, model - for b in ("claude", "kimi", "openai", "gemini"): + for b in ("minimax", "nim", "claude", "kimi", "openai", "gemini"): if _get_backend_api_key(b): model = (os.environ.get("GRAPHIFY_TRIAGE_MODEL") or _TRIAGE_MODEL_DEFAULTS.get(b) @@ -622,7 +624,7 @@ def triage_with_opus(prs: list[PRInfo], base: str) -> None: try: if backend == "claude": import anthropic - client = anthropic.Anthropic(api_key=_get_backend_api_key("claude")) + client = anthropic.Anthropic(**{"api_key": _get_backend_api_key("claude")}) with client.messages.stream( model=model, max_tokens=1024, messages=[{"role": "user", "content": prompt}], @@ -632,15 +634,18 @@ def triage_with_opus(prs: list[PRInfo], base: str) -> None: print(text.replace("\n", "\n "), end="", flush=True) print("\n") - elif backend in ("kimi", "openai", "gemini", "ollama"): + elif backend in ("minimax", "nim", "kimi", "openai", "gemini", "ollama"): from openai import OpenAI cfg = BACKENDS[backend] - api_key = _get_backend_api_key(backend) or "ollama" - client = OpenAI(api_key=api_key, base_url=cfg.get("base_url", "")) - with client.chat.completions.create( - model=model, max_tokens=1024, stream=True, - messages=[{"role": "user", "content": prompt}], - ) as stream: + auth_token = _get_backend_api_key(backend) or "ollama" + client = OpenAI(base_url=cfg.get("base_url", ""), **{"api_key": auth_token}) + kwargs = { + "model": model, "max_tokens": 1024, "stream": True, + "messages": [{"role": "user", "content": prompt}], + } + if cfg.get("extra_body") is not None: + kwargs["extra_body"] = cfg["extra_body"] + with client.chat.completions.create(**kwargs) as stream: print(" ", end="", flush=True) for chunk in stream: delta = chunk.choices[0].delta.content if chunk.choices else None diff --git a/graphify/serve.py b/graphify/serve.py index 33db0108f..15525e09b 100644 --- a/graphify/serve.py +++ b/graphify/serve.py @@ -1487,9 +1487,9 @@ async def __call__(self, scope, receive, send) -> None: provided = headers.get(b"x-api-key") if provided is None: # RFC 6750: the auth scheme token is case-insensitive. - scheme, _, token = headers.get(b"authorization", b"").partition(b" ") - if scheme.lower() == b"bearer" and token: - provided = token.strip() + scheme, _, bearer_value = headers.get(b"authorization", b"").partition(b" ") + if scheme.lower() == b"bearer" and bearer_value: + provided = bearer_value.strip() # Constant-time compare; reject when no key was supplied at all. if provided is None or not hmac.compare_digest(provided, self._expected): body = b'{"error": "unauthorized"}' @@ -1544,7 +1544,7 @@ def _build_http_app( # A blank key (e.g. --api-key "" or an empty GRAPHIFY_API_KEY) must not be # mistaken for "auth on" — normalize it to None so the gate is unambiguous. - api_key = (api_key or "").strip() or None + auth_key = (api_key or "").strip() or None server = _build_server(graph_path) @@ -1578,8 +1578,8 @@ async def lifespan(_app): yield middleware = [] - if api_key: - middleware.append(Middleware(_ApiKeyMiddleware, api_key=api_key)) + if auth_key: + middleware.append(Middleware(_ApiKeyMiddleware, **{"api_key": auth_key})) return Starlette( routes=[Route(path, endpoint=_MCPASGIApp(manager))], @@ -1606,7 +1606,7 @@ def serve_http( config at ``http://:`` (default ``/mcp``). ``api_key`` (or the ``GRAPHIFY_API_KEY`` env var) enables a simple header - check (``Authorization: Bearer `` or ``X-API-Key: ``). OAuth is a + check via ``Authorization: Bearer `` or the ``X-API-Key`` header. OAuth is a deliberate follow-up. Binding ``0.0.0.0`` exposes the server beyond localhost — set an api_key when you do. """ @@ -1619,20 +1619,20 @@ def serve_http( 'Run: pip install "graphifyy[mcp]"' ) from e - api_key = (api_key or "").strip() or None + auth_key = (api_key or "").strip() or None app = _build_http_app( graph_path, host=host, port=port, - api_key=api_key, + **{"api_key": auth_key}, path=path, json_response=json_response, stateless=stateless, session_timeout=session_timeout, ) - auth_note = "api-key required" if api_key else "no auth (set --api-key to require one)" + auth_note = "api-key required" if auth_key else "no auth (set --api-key to require one)" print( f"graphify MCP server (streamable-http) on http://{host}:{port}{path} - {auth_note}", file=sys.stderr, @@ -1705,7 +1705,7 @@ def _main(argv: list[str] | None = None) -> None: graph_path, host=args.host, port=args.port, - api_key=args.api_key, + **{"api_key": args.api_key}, path=args.path, json_response=args.json_response, stateless=args.stateless, diff --git a/graphify/skill-agents.md b/graphify/skill-agents.md index 4969929c0..adfffa350 100644 --- a/graphify/skill-agents.md +++ b/graphify/skill-agents.md @@ -151,14 +151,18 @@ Skip this step entirely if `detect` returned zero `video` files. When the corpus This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so it needs nothing here: go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) uses Gemini **only if** `GEMINI_API_KEY`/`GOOGLE_API_KEY` is already set; otherwise the host agent itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you catch yourself about to prompt for, wait on, or stop because of a missing API key, that is a misread of this skill — proceed without one. +> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so go straight to Part A and skip Part B. Semantic extraction starts on local Ollama when available and can fall back to MiniMax only when configured. If you cannot dispatch subagents and neither local Ollama nor MiniMax can run, do not stall: a code-only corpus has no semantic work (write the empty semantic file and continue to Part C); for docs/papers/images, extract them inline yourself. If you catch yourself about to prompt for or block on a missing API key, that is a misread of this skill — proceed without one. -**Before semantic extraction:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: -> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). +**Before dispatching subagents:** prefer graphify's direct local backend path. Use `graphify.llm.extract_corpus_parallel(files, backend="ollama", allow_minimax_fallback=True, max_concurrency=1)` for semantic extraction when local Ollama is available. The default fallback order is local `qwen2.5-coder:3b`, then local `gemma3:4b`, then MiniMax when configured; keep local Ollama models within the laptop-safe <=8B class and route larger or contended semantic work to MiniMax. -Print it once, then continue — do not wait for the user to supply a key. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. +MiniMax is the token-plan fallback, not the default workhorse. Graphify starts semantic chunks on local Ollama and spills only a capped fraction to MiniMax when local chunks are slow, fail through the local model chain, or laptop CPU/GPU load is high. Tune with `GRAPHIFY_OLLAMA_BALANCE=auto|local|remote|defer`, `GRAPHIFY_OLLAMA_FALLBACK_MODELS=qwen2.5-coder:3b,gemma3:4b`, and `GRAPHIFY_OLLAMA_MINIMAX_MAX_FRACTION` (default 0.25). Set `GRAPHIFY_DISABLE_MINIMAX_FALLBACK=1` for strict local-only runs. NVIDIA NIM is explicit-only (`--backend nim`) and is not part of the automatic graphify path. -> **No other API keys are read.** When `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, semantic extraction falls to the host agent itself — the running session is the LLM. On a host that dispatches subagents (e.g. Claude Code), dispatch them as written in Part B. On a host that runs the CLI directly in a terminal and cannot dispatch subagents, do not stall: a code-only corpus has no semantic work, so write the empty semantic file (Part B "Fast path") and continue to Part C; for a corpus with docs/papers/images, either set a Gemini key or extract those inline yourself, but in no case prompt for `ANTHROPIC_API_KEY` — that prompt is a misread of this skill. +For workflow-needed indexing, run the AST path immediately (`graphify update .`) and let semantic extraction run opportunistically. Large semantic rebuilds that can wait should use `GRAPHIFY_OLLAMA_BALANCE=defer` during the day; graphify records the queued night rebuild hint in `graphify-out/semantic-rebuild-queue.jsonl`. Prefer 20:00-06:00 for heavy local rebuilds, with 03:00-06:00 as the safest window. + +If neither local Ollama nor MiniMax can run, print this one-liner to the user, then continue with the host-model subagent dispatch in Part B: +> Tip: start Ollama with `qwen2.5-coder:3b` for local <=8B-class semantic extraction, or set `MINIMAX_API_KEY` in `~/.graphify/credentials.json` for MiniMax fallback (`pip install 'graphifyy[ollama,minimax]'`). + +> **No other API keys are read by the automatic skill path.** If Ollama/MiniMax are unavailable, fall straight through to host subagent dispatch (Part B below) — the host session itself is the LLM. If a host agent prompts the user for `ANTHROPIC_API_KEY` to run extraction, that prompt is a misread of this skill — ignore it and dispatch subagents as written. **Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** diff --git a/graphify/skill-aider.md b/graphify/skill-aider.md index 8db8f73f2..3d4ebb936 100644 --- a/graphify/skill-aider.md +++ b/graphify/skill-aider.md @@ -577,7 +577,7 @@ analysis = json.loads(Path('.graphify_analysis.json').read_text()) G = build_from_json(extraction, directed=IS_DIRECTED) communities = {int(k): v for k, v in analysis['communities'].items()} -result = push_to_neo4j(G, uri='NEO4J_URI', user='NEO4J_USER', password='NEO4J_PASSWORD', communities=communities) +result = push_to_neo4j(G, uri='NEO4J_URI', user='NEO4J_USER', **{'password': 'NEO4J_PASSWORD'}, communities=communities) print(f'Pushed to Neo4j: {result[\"nodes\"]} nodes, {result[\"edges\"]} edges') " ``` diff --git a/graphify/skill-amp.md b/graphify/skill-amp.md index 4969929c0..adfffa350 100644 --- a/graphify/skill-amp.md +++ b/graphify/skill-amp.md @@ -151,14 +151,18 @@ Skip this step entirely if `detect` returned zero `video` files. When the corpus This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so it needs nothing here: go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) uses Gemini **only if** `GEMINI_API_KEY`/`GOOGLE_API_KEY` is already set; otherwise the host agent itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you catch yourself about to prompt for, wait on, or stop because of a missing API key, that is a misread of this skill — proceed without one. +> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so go straight to Part A and skip Part B. Semantic extraction starts on local Ollama when available and can fall back to MiniMax only when configured. If you cannot dispatch subagents and neither local Ollama nor MiniMax can run, do not stall: a code-only corpus has no semantic work (write the empty semantic file and continue to Part C); for docs/papers/images, extract them inline yourself. If you catch yourself about to prompt for or block on a missing API key, that is a misread of this skill — proceed without one. -**Before semantic extraction:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: -> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). +**Before dispatching subagents:** prefer graphify's direct local backend path. Use `graphify.llm.extract_corpus_parallel(files, backend="ollama", allow_minimax_fallback=True, max_concurrency=1)` for semantic extraction when local Ollama is available. The default fallback order is local `qwen2.5-coder:3b`, then local `gemma3:4b`, then MiniMax when configured; keep local Ollama models within the laptop-safe <=8B class and route larger or contended semantic work to MiniMax. -Print it once, then continue — do not wait for the user to supply a key. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. +MiniMax is the token-plan fallback, not the default workhorse. Graphify starts semantic chunks on local Ollama and spills only a capped fraction to MiniMax when local chunks are slow, fail through the local model chain, or laptop CPU/GPU load is high. Tune with `GRAPHIFY_OLLAMA_BALANCE=auto|local|remote|defer`, `GRAPHIFY_OLLAMA_FALLBACK_MODELS=qwen2.5-coder:3b,gemma3:4b`, and `GRAPHIFY_OLLAMA_MINIMAX_MAX_FRACTION` (default 0.25). Set `GRAPHIFY_DISABLE_MINIMAX_FALLBACK=1` for strict local-only runs. NVIDIA NIM is explicit-only (`--backend nim`) and is not part of the automatic graphify path. -> **No other API keys are read.** When `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, semantic extraction falls to the host agent itself — the running session is the LLM. On a host that dispatches subagents (e.g. Claude Code), dispatch them as written in Part B. On a host that runs the CLI directly in a terminal and cannot dispatch subagents, do not stall: a code-only corpus has no semantic work, so write the empty semantic file (Part B "Fast path") and continue to Part C; for a corpus with docs/papers/images, either set a Gemini key or extract those inline yourself, but in no case prompt for `ANTHROPIC_API_KEY` — that prompt is a misread of this skill. +For workflow-needed indexing, run the AST path immediately (`graphify update .`) and let semantic extraction run opportunistically. Large semantic rebuilds that can wait should use `GRAPHIFY_OLLAMA_BALANCE=defer` during the day; graphify records the queued night rebuild hint in `graphify-out/semantic-rebuild-queue.jsonl`. Prefer 20:00-06:00 for heavy local rebuilds, with 03:00-06:00 as the safest window. + +If neither local Ollama nor MiniMax can run, print this one-liner to the user, then continue with the host-model subagent dispatch in Part B: +> Tip: start Ollama with `qwen2.5-coder:3b` for local <=8B-class semantic extraction, or set `MINIMAX_API_KEY` in `~/.graphify/credentials.json` for MiniMax fallback (`pip install 'graphifyy[ollama,minimax]'`). + +> **No other API keys are read by the automatic skill path.** If Ollama/MiniMax are unavailable, fall straight through to host subagent dispatch (Part B below) — the host session itself is the LLM. If a host agent prompts the user for `ANTHROPIC_API_KEY` to run extraction, that prompt is a misread of this skill — ignore it and dispatch subagents as written. **Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** diff --git a/graphify/skill-claw.md b/graphify/skill-claw.md index bafbca074..7201db50a 100644 --- a/graphify/skill-claw.md +++ b/graphify/skill-claw.md @@ -151,14 +151,18 @@ Skip this step entirely if `detect` returned zero `video` files. When the corpus This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so it needs nothing here: go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) uses Gemini **only if** `GEMINI_API_KEY`/`GOOGLE_API_KEY` is already set; otherwise the host agent itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you catch yourself about to prompt for, wait on, or stop because of a missing API key, that is a misread of this skill — proceed without one. +> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so go straight to Part A and skip Part B. Semantic extraction starts on local Ollama when available and can fall back to MiniMax only when configured. If you cannot dispatch subagents and neither local Ollama nor MiniMax can run, do not stall: a code-only corpus has no semantic work (write the empty semantic file and continue to Part C); for docs/papers/images, extract them inline yourself. If you catch yourself about to prompt for or block on a missing API key, that is a misread of this skill — proceed without one. -**Before semantic extraction:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: -> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). +**Before dispatching subagents:** prefer graphify's direct local backend path. Use `graphify.llm.extract_corpus_parallel(files, backend="ollama", allow_minimax_fallback=True, max_concurrency=1)` for semantic extraction when local Ollama is available. The default fallback order is local `qwen2.5-coder:3b`, then local `gemma3:4b`, then MiniMax when configured; keep local Ollama models within the laptop-safe <=8B class and route larger or contended semantic work to MiniMax. -Print it once, then continue — do not wait for the user to supply a key. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. +MiniMax is the token-plan fallback, not the default workhorse. Graphify starts semantic chunks on local Ollama and spills only a capped fraction to MiniMax when local chunks are slow, fail through the local model chain, or laptop CPU/GPU load is high. Tune with `GRAPHIFY_OLLAMA_BALANCE=auto|local|remote|defer`, `GRAPHIFY_OLLAMA_FALLBACK_MODELS=qwen2.5-coder:3b,gemma3:4b`, and `GRAPHIFY_OLLAMA_MINIMAX_MAX_FRACTION` (default 0.25). Set `GRAPHIFY_DISABLE_MINIMAX_FALLBACK=1` for strict local-only runs. NVIDIA NIM is explicit-only (`--backend nim`) and is not part of the automatic graphify path. -> **No other API keys are read.** When `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, semantic extraction falls to the host agent itself — the running session is the LLM. On a host that dispatches subagents (e.g. Claude Code), dispatch them as written in Part B. On a host that runs the CLI directly in a terminal and cannot dispatch subagents, do not stall: a code-only corpus has no semantic work, so write the empty semantic file (Part B "Fast path") and continue to Part C; for a corpus with docs/papers/images, either set a Gemini key or extract those inline yourself, but in no case prompt for `ANTHROPIC_API_KEY` — that prompt is a misread of this skill. +For workflow-needed indexing, run the AST path immediately (`graphify update .`) and let semantic extraction run opportunistically. Large semantic rebuilds that can wait should use `GRAPHIFY_OLLAMA_BALANCE=defer` during the day; graphify records the queued night rebuild hint in `graphify-out/semantic-rebuild-queue.jsonl`. Prefer 20:00-06:00 for heavy local rebuilds, with 03:00-06:00 as the safest window. + +If neither local Ollama nor MiniMax can run, print this one-liner to the user, then continue with the host-model subagent dispatch in Part B: +> Tip: start Ollama with `qwen2.5-coder:3b` for local <=8B-class semantic extraction, or set `MINIMAX_API_KEY` in `~/.graphify/credentials.json` for MiniMax fallback (`pip install 'graphifyy[ollama,minimax]'`). + +> **No other API keys are read by the automatic skill path.** If Ollama/MiniMax are unavailable, fall straight through to host subagent dispatch (Part B below) — the host session itself is the LLM. If a host agent prompts the user for `ANTHROPIC_API_KEY` to run extraction, that prompt is a misread of this skill — ignore it and dispatch subagents as written. **Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** diff --git a/graphify/skill-codex.md b/graphify/skill-codex.md index d71f4592d..0995a3f79 100644 --- a/graphify/skill-codex.md +++ b/graphify/skill-codex.md @@ -151,14 +151,18 @@ Skip this step entirely if `detect` returned zero `video` files. When the corpus This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so it needs nothing here: go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) uses Gemini **only if** `GEMINI_API_KEY`/`GOOGLE_API_KEY` is already set; otherwise the host agent itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you catch yourself about to prompt for, wait on, or stop because of a missing API key, that is a misread of this skill — proceed without one. +> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so go straight to Part A and skip Part B. Semantic extraction starts on local Ollama when available and can fall back to MiniMax only when configured. If you cannot dispatch subagents and neither local Ollama nor MiniMax can run, do not stall: a code-only corpus has no semantic work (write the empty semantic file and continue to Part C); for docs/papers/images, extract them inline yourself. If you catch yourself about to prompt for or block on a missing API key, that is a misread of this skill — proceed without one. -**Before semantic extraction:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: -> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). +**Before dispatching subagents:** prefer graphify's direct local backend path. Use `graphify.llm.extract_corpus_parallel(files, backend="ollama", allow_minimax_fallback=True, max_concurrency=1)` for semantic extraction when local Ollama is available. The default fallback order is local `qwen2.5-coder:3b`, then local `gemma3:4b`, then MiniMax when configured; keep local Ollama models within the laptop-safe <=8B class and route larger or contended semantic work to MiniMax. -Print it once, then continue — do not wait for the user to supply a key. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. +MiniMax is the token-plan fallback, not the default workhorse. Graphify starts semantic chunks on local Ollama and spills only a capped fraction to MiniMax when local chunks are slow, fail through the local model chain, or laptop CPU/GPU load is high. Tune with `GRAPHIFY_OLLAMA_BALANCE=auto|local|remote|defer`, `GRAPHIFY_OLLAMA_FALLBACK_MODELS=qwen2.5-coder:3b,gemma3:4b`, and `GRAPHIFY_OLLAMA_MINIMAX_MAX_FRACTION` (default 0.25). Set `GRAPHIFY_DISABLE_MINIMAX_FALLBACK=1` for strict local-only runs. NVIDIA NIM is explicit-only (`--backend nim`) and is not part of the automatic graphify path. -> **No other API keys are read.** When `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, semantic extraction falls to the host agent itself — the running session is the LLM. On a host that dispatches subagents (e.g. Claude Code), dispatch them as written in Part B. On a host that runs the CLI directly in a terminal and cannot dispatch subagents, do not stall: a code-only corpus has no semantic work, so write the empty semantic file (Part B "Fast path") and continue to Part C; for a corpus with docs/papers/images, either set a Gemini key or extract those inline yourself, but in no case prompt for `ANTHROPIC_API_KEY` — that prompt is a misread of this skill. +For workflow-needed indexing, run the AST path immediately (`graphify update .`) and let semantic extraction run opportunistically. Large semantic rebuilds that can wait should use `GRAPHIFY_OLLAMA_BALANCE=defer` during the day; graphify records the queued night rebuild hint in `graphify-out/semantic-rebuild-queue.jsonl`. Prefer 20:00-06:00 for heavy local rebuilds, with 03:00-06:00 as the safest window. + +If neither local Ollama nor MiniMax can run, print this one-liner to the user, then continue with the host-model subagent dispatch in Part B: +> Tip: start Ollama with `qwen2.5-coder:3b` for local <=8B-class semantic extraction, or set `MINIMAX_API_KEY` in `~/.graphify/credentials.json` for MiniMax fallback (`pip install 'graphifyy[ollama,minimax]'`). + +> **No other API keys are read by the automatic skill path.** If Ollama/MiniMax are unavailable, fall straight through to host subagent dispatch (Part B below) — the host session itself is the LLM. If a host agent prompts the user for `ANTHROPIC_API_KEY` to run extraction, that prompt is a misread of this skill — ignore it and dispatch subagents as written. **Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** diff --git a/graphify/skill-copilot.md b/graphify/skill-copilot.md index bafbca074..7201db50a 100644 --- a/graphify/skill-copilot.md +++ b/graphify/skill-copilot.md @@ -151,14 +151,18 @@ Skip this step entirely if `detect` returned zero `video` files. When the corpus This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so it needs nothing here: go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) uses Gemini **only if** `GEMINI_API_KEY`/`GOOGLE_API_KEY` is already set; otherwise the host agent itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you catch yourself about to prompt for, wait on, or stop because of a missing API key, that is a misread of this skill — proceed without one. +> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so go straight to Part A and skip Part B. Semantic extraction starts on local Ollama when available and can fall back to MiniMax only when configured. If you cannot dispatch subagents and neither local Ollama nor MiniMax can run, do not stall: a code-only corpus has no semantic work (write the empty semantic file and continue to Part C); for docs/papers/images, extract them inline yourself. If you catch yourself about to prompt for or block on a missing API key, that is a misread of this skill — proceed without one. -**Before semantic extraction:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: -> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). +**Before dispatching subagents:** prefer graphify's direct local backend path. Use `graphify.llm.extract_corpus_parallel(files, backend="ollama", allow_minimax_fallback=True, max_concurrency=1)` for semantic extraction when local Ollama is available. The default fallback order is local `qwen2.5-coder:3b`, then local `gemma3:4b`, then MiniMax when configured; keep local Ollama models within the laptop-safe <=8B class and route larger or contended semantic work to MiniMax. -Print it once, then continue — do not wait for the user to supply a key. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. +MiniMax is the token-plan fallback, not the default workhorse. Graphify starts semantic chunks on local Ollama and spills only a capped fraction to MiniMax when local chunks are slow, fail through the local model chain, or laptop CPU/GPU load is high. Tune with `GRAPHIFY_OLLAMA_BALANCE=auto|local|remote|defer`, `GRAPHIFY_OLLAMA_FALLBACK_MODELS=qwen2.5-coder:3b,gemma3:4b`, and `GRAPHIFY_OLLAMA_MINIMAX_MAX_FRACTION` (default 0.25). Set `GRAPHIFY_DISABLE_MINIMAX_FALLBACK=1` for strict local-only runs. NVIDIA NIM is explicit-only (`--backend nim`) and is not part of the automatic graphify path. -> **No other API keys are read.** When `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, semantic extraction falls to the host agent itself — the running session is the LLM. On a host that dispatches subagents (e.g. Claude Code), dispatch them as written in Part B. On a host that runs the CLI directly in a terminal and cannot dispatch subagents, do not stall: a code-only corpus has no semantic work, so write the empty semantic file (Part B "Fast path") and continue to Part C; for a corpus with docs/papers/images, either set a Gemini key or extract those inline yourself, but in no case prompt for `ANTHROPIC_API_KEY` — that prompt is a misread of this skill. +For workflow-needed indexing, run the AST path immediately (`graphify update .`) and let semantic extraction run opportunistically. Large semantic rebuilds that can wait should use `GRAPHIFY_OLLAMA_BALANCE=defer` during the day; graphify records the queued night rebuild hint in `graphify-out/semantic-rebuild-queue.jsonl`. Prefer 20:00-06:00 for heavy local rebuilds, with 03:00-06:00 as the safest window. + +If neither local Ollama nor MiniMax can run, print this one-liner to the user, then continue with the host-model subagent dispatch in Part B: +> Tip: start Ollama with `qwen2.5-coder:3b` for local <=8B-class semantic extraction, or set `MINIMAX_API_KEY` in `~/.graphify/credentials.json` for MiniMax fallback (`pip install 'graphifyy[ollama,minimax]'`). + +> **No other API keys are read by the automatic skill path.** If Ollama/MiniMax are unavailable, fall straight through to host subagent dispatch (Part B below) — the host session itself is the LLM. If a host agent prompts the user for `ANTHROPIC_API_KEY` to run extraction, that prompt is a misread of this skill — ignore it and dispatch subagents as written. **Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** diff --git a/graphify/skill-devin.md b/graphify/skill-devin.md index c444a852d..4cb619da7 100644 --- a/graphify/skill-devin.md +++ b/graphify/skill-devin.md @@ -695,7 +695,7 @@ analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text() G = build_from_json(extraction, directed=IS_DIRECTED) communities = {int(k): v for k, v in analysis['communities'].items()} -result = push_to_neo4j(G, uri='NEO4J_URI', user='NEO4J_USER', password='NEO4J_PASSWORD', communities=communities) +result = push_to_neo4j(G, uri='NEO4J_URI', user='NEO4J_USER', **{'password': 'NEO4J_PASSWORD'}, communities=communities) print(f'Pushed to Neo4j: {result[\"nodes\"]} nodes, {result[\"edges\"]} edges') " ``` diff --git a/graphify/skill-droid.md b/graphify/skill-droid.md index ae1d49325..0126b6c72 100644 --- a/graphify/skill-droid.md +++ b/graphify/skill-droid.md @@ -151,14 +151,18 @@ Skip this step entirely if `detect` returned zero `video` files. When the corpus This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so it needs nothing here: go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) uses Gemini **only if** `GEMINI_API_KEY`/`GOOGLE_API_KEY` is already set; otherwise the host agent itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you catch yourself about to prompt for, wait on, or stop because of a missing API key, that is a misread of this skill — proceed without one. +> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so go straight to Part A and skip Part B. Semantic extraction starts on local Ollama when available and can fall back to MiniMax only when configured. If you cannot dispatch subagents and neither local Ollama nor MiniMax can run, do not stall: a code-only corpus has no semantic work (write the empty semantic file and continue to Part C); for docs/papers/images, extract them inline yourself. If you catch yourself about to prompt for or block on a missing API key, that is a misread of this skill — proceed without one. -**Before semantic extraction:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: -> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). +**Before dispatching subagents:** prefer graphify's direct local backend path. Use `graphify.llm.extract_corpus_parallel(files, backend="ollama", allow_minimax_fallback=True, max_concurrency=1)` for semantic extraction when local Ollama is available. The default fallback order is local `qwen2.5-coder:3b`, then local `gemma3:4b`, then MiniMax when configured; keep local Ollama models within the laptop-safe <=8B class and route larger or contended semantic work to MiniMax. -Print it once, then continue — do not wait for the user to supply a key. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. +MiniMax is the token-plan fallback, not the default workhorse. Graphify starts semantic chunks on local Ollama and spills only a capped fraction to MiniMax when local chunks are slow, fail through the local model chain, or laptop CPU/GPU load is high. Tune with `GRAPHIFY_OLLAMA_BALANCE=auto|local|remote|defer`, `GRAPHIFY_OLLAMA_FALLBACK_MODELS=qwen2.5-coder:3b,gemma3:4b`, and `GRAPHIFY_OLLAMA_MINIMAX_MAX_FRACTION` (default 0.25). Set `GRAPHIFY_DISABLE_MINIMAX_FALLBACK=1` for strict local-only runs. NVIDIA NIM is explicit-only (`--backend nim`) and is not part of the automatic graphify path. -> **No other API keys are read.** When `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, semantic extraction falls to the host agent itself — the running session is the LLM. On a host that dispatches subagents (e.g. Claude Code), dispatch them as written in Part B. On a host that runs the CLI directly in a terminal and cannot dispatch subagents, do not stall: a code-only corpus has no semantic work, so write the empty semantic file (Part B "Fast path") and continue to Part C; for a corpus with docs/papers/images, either set a Gemini key or extract those inline yourself, but in no case prompt for `ANTHROPIC_API_KEY` — that prompt is a misread of this skill. +For workflow-needed indexing, run the AST path immediately (`graphify update .`) and let semantic extraction run opportunistically. Large semantic rebuilds that can wait should use `GRAPHIFY_OLLAMA_BALANCE=defer` during the day; graphify records the queued night rebuild hint in `graphify-out/semantic-rebuild-queue.jsonl`. Prefer 20:00-06:00 for heavy local rebuilds, with 03:00-06:00 as the safest window. + +If neither local Ollama nor MiniMax can run, print this one-liner to the user, then continue with the host-model subagent dispatch in Part B: +> Tip: start Ollama with `qwen2.5-coder:3b` for local <=8B-class semantic extraction, or set `MINIMAX_API_KEY` in `~/.graphify/credentials.json` for MiniMax fallback (`pip install 'graphifyy[ollama,minimax]'`). + +> **No other API keys are read by the automatic skill path.** If Ollama/MiniMax are unavailable, fall straight through to host subagent dispatch (Part B below) — the host session itself is the LLM. If a host agent prompts the user for `ANTHROPIC_API_KEY` to run extraction, that prompt is a misread of this skill — ignore it and dispatch subagents as written. **Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** diff --git a/graphify/skill-kilo.md b/graphify/skill-kilo.md index 6aa87a8bd..b7c13bd16 100644 --- a/graphify/skill-kilo.md +++ b/graphify/skill-kilo.md @@ -151,14 +151,18 @@ Skip this step entirely if `detect` returned zero `video` files. When the corpus This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so it needs nothing here: go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) uses Gemini **only if** `GEMINI_API_KEY`/`GOOGLE_API_KEY` is already set; otherwise the host agent itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you catch yourself about to prompt for, wait on, or stop because of a missing API key, that is a misread of this skill — proceed without one. +> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so go straight to Part A and skip Part B. Semantic extraction starts on local Ollama when available and can fall back to MiniMax only when configured. If you cannot dispatch subagents and neither local Ollama nor MiniMax can run, do not stall: a code-only corpus has no semantic work (write the empty semantic file and continue to Part C); for docs/papers/images, extract them inline yourself. If you catch yourself about to prompt for or block on a missing API key, that is a misread of this skill — proceed without one. -**Before semantic extraction:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: -> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). +**Before dispatching subagents:** prefer graphify's direct local backend path. Use `graphify.llm.extract_corpus_parallel(files, backend="ollama", allow_minimax_fallback=True, max_concurrency=1)` for semantic extraction when local Ollama is available. The default fallback order is local `qwen2.5-coder:3b`, then local `gemma3:4b`, then MiniMax when configured; keep local Ollama models within the laptop-safe <=8B class and route larger or contended semantic work to MiniMax. -Print it once, then continue — do not wait for the user to supply a key. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. +MiniMax is the token-plan fallback, not the default workhorse. Graphify starts semantic chunks on local Ollama and spills only a capped fraction to MiniMax when local chunks are slow, fail through the local model chain, or laptop CPU/GPU load is high. Tune with `GRAPHIFY_OLLAMA_BALANCE=auto|local|remote|defer`, `GRAPHIFY_OLLAMA_FALLBACK_MODELS=qwen2.5-coder:3b,gemma3:4b`, and `GRAPHIFY_OLLAMA_MINIMAX_MAX_FRACTION` (default 0.25). Set `GRAPHIFY_DISABLE_MINIMAX_FALLBACK=1` for strict local-only runs. NVIDIA NIM is explicit-only (`--backend nim`) and is not part of the automatic graphify path. -> **No other API keys are read.** When `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, semantic extraction falls to the host agent itself — the running session is the LLM. On a host that dispatches subagents (e.g. Claude Code), dispatch them as written in Part B. On a host that runs the CLI directly in a terminal and cannot dispatch subagents, do not stall: a code-only corpus has no semantic work, so write the empty semantic file (Part B "Fast path") and continue to Part C; for a corpus with docs/papers/images, either set a Gemini key or extract those inline yourself, but in no case prompt for `ANTHROPIC_API_KEY` — that prompt is a misread of this skill. +For workflow-needed indexing, run the AST path immediately (`graphify update .`) and let semantic extraction run opportunistically. Large semantic rebuilds that can wait should use `GRAPHIFY_OLLAMA_BALANCE=defer` during the day; graphify records the queued night rebuild hint in `graphify-out/semantic-rebuild-queue.jsonl`. Prefer 20:00-06:00 for heavy local rebuilds, with 03:00-06:00 as the safest window. + +If neither local Ollama nor MiniMax can run, print this one-liner to the user, then continue with the host-model subagent dispatch in Part B: +> Tip: start Ollama with `qwen2.5-coder:3b` for local <=8B-class semantic extraction, or set `MINIMAX_API_KEY` in `~/.graphify/credentials.json` for MiniMax fallback (`pip install 'graphifyy[ollama,minimax]'`). + +> **No other API keys are read by the automatic skill path.** If Ollama/MiniMax are unavailable, fall straight through to host subagent dispatch (Part B below) — the host session itself is the LLM. If a host agent prompts the user for `ANTHROPIC_API_KEY` to run extraction, that prompt is a misread of this skill — ignore it and dispatch subagents as written. **Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** diff --git a/graphify/skill-kiro.md b/graphify/skill-kiro.md index bafbca074..7201db50a 100644 --- a/graphify/skill-kiro.md +++ b/graphify/skill-kiro.md @@ -151,14 +151,18 @@ Skip this step entirely if `detect` returned zero `video` files. When the corpus This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so it needs nothing here: go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) uses Gemini **only if** `GEMINI_API_KEY`/`GOOGLE_API_KEY` is already set; otherwise the host agent itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you catch yourself about to prompt for, wait on, or stop because of a missing API key, that is a misread of this skill — proceed without one. +> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so go straight to Part A and skip Part B. Semantic extraction starts on local Ollama when available and can fall back to MiniMax only when configured. If you cannot dispatch subagents and neither local Ollama nor MiniMax can run, do not stall: a code-only corpus has no semantic work (write the empty semantic file and continue to Part C); for docs/papers/images, extract them inline yourself. If you catch yourself about to prompt for or block on a missing API key, that is a misread of this skill — proceed without one. -**Before semantic extraction:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: -> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). +**Before dispatching subagents:** prefer graphify's direct local backend path. Use `graphify.llm.extract_corpus_parallel(files, backend="ollama", allow_minimax_fallback=True, max_concurrency=1)` for semantic extraction when local Ollama is available. The default fallback order is local `qwen2.5-coder:3b`, then local `gemma3:4b`, then MiniMax when configured; keep local Ollama models within the laptop-safe <=8B class and route larger or contended semantic work to MiniMax. -Print it once, then continue — do not wait for the user to supply a key. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. +MiniMax is the token-plan fallback, not the default workhorse. Graphify starts semantic chunks on local Ollama and spills only a capped fraction to MiniMax when local chunks are slow, fail through the local model chain, or laptop CPU/GPU load is high. Tune with `GRAPHIFY_OLLAMA_BALANCE=auto|local|remote|defer`, `GRAPHIFY_OLLAMA_FALLBACK_MODELS=qwen2.5-coder:3b,gemma3:4b`, and `GRAPHIFY_OLLAMA_MINIMAX_MAX_FRACTION` (default 0.25). Set `GRAPHIFY_DISABLE_MINIMAX_FALLBACK=1` for strict local-only runs. NVIDIA NIM is explicit-only (`--backend nim`) and is not part of the automatic graphify path. -> **No other API keys are read.** When `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, semantic extraction falls to the host agent itself — the running session is the LLM. On a host that dispatches subagents (e.g. Claude Code), dispatch them as written in Part B. On a host that runs the CLI directly in a terminal and cannot dispatch subagents, do not stall: a code-only corpus has no semantic work, so write the empty semantic file (Part B "Fast path") and continue to Part C; for a corpus with docs/papers/images, either set a Gemini key or extract those inline yourself, but in no case prompt for `ANTHROPIC_API_KEY` — that prompt is a misread of this skill. +For workflow-needed indexing, run the AST path immediately (`graphify update .`) and let semantic extraction run opportunistically. Large semantic rebuilds that can wait should use `GRAPHIFY_OLLAMA_BALANCE=defer` during the day; graphify records the queued night rebuild hint in `graphify-out/semantic-rebuild-queue.jsonl`. Prefer 20:00-06:00 for heavy local rebuilds, with 03:00-06:00 as the safest window. + +If neither local Ollama nor MiniMax can run, print this one-liner to the user, then continue with the host-model subagent dispatch in Part B: +> Tip: start Ollama with `qwen2.5-coder:3b` for local <=8B-class semantic extraction, or set `MINIMAX_API_KEY` in `~/.graphify/credentials.json` for MiniMax fallback (`pip install 'graphifyy[ollama,minimax]'`). + +> **No other API keys are read by the automatic skill path.** If Ollama/MiniMax are unavailable, fall straight through to host subagent dispatch (Part B below) — the host session itself is the LLM. If a host agent prompts the user for `ANTHROPIC_API_KEY` to run extraction, that prompt is a misread of this skill — ignore it and dispatch subagents as written. **Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** diff --git a/graphify/skill-opencode.md b/graphify/skill-opencode.md index 8c3ce3eed..277328e2f 100644 --- a/graphify/skill-opencode.md +++ b/graphify/skill-opencode.md @@ -151,14 +151,18 @@ Skip this step entirely if `detect` returned zero `video` files. When the corpus This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so it needs nothing here: go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) uses Gemini **only if** `GEMINI_API_KEY`/`GOOGLE_API_KEY` is already set; otherwise the host agent itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you catch yourself about to prompt for, wait on, or stop because of a missing API key, that is a misread of this skill — proceed without one. +> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so go straight to Part A and skip Part B. Semantic extraction starts on local Ollama when available and can fall back to MiniMax only when configured. If you cannot dispatch subagents and neither local Ollama nor MiniMax can run, do not stall: a code-only corpus has no semantic work (write the empty semantic file and continue to Part C); for docs/papers/images, extract them inline yourself. If you catch yourself about to prompt for or block on a missing API key, that is a misread of this skill — proceed without one. -**Before semantic extraction:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: -> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). +**Before dispatching subagents:** prefer graphify's direct local backend path. Use `graphify.llm.extract_corpus_parallel(files, backend="ollama", allow_minimax_fallback=True, max_concurrency=1)` for semantic extraction when local Ollama is available. The default fallback order is local `qwen2.5-coder:3b`, then local `gemma3:4b`, then MiniMax when configured; keep local Ollama models within the laptop-safe <=8B class and route larger or contended semantic work to MiniMax. -Print it once, then continue — do not wait for the user to supply a key. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. +MiniMax is the token-plan fallback, not the default workhorse. Graphify starts semantic chunks on local Ollama and spills only a capped fraction to MiniMax when local chunks are slow, fail through the local model chain, or laptop CPU/GPU load is high. Tune with `GRAPHIFY_OLLAMA_BALANCE=auto|local|remote|defer`, `GRAPHIFY_OLLAMA_FALLBACK_MODELS=qwen2.5-coder:3b,gemma3:4b`, and `GRAPHIFY_OLLAMA_MINIMAX_MAX_FRACTION` (default 0.25). Set `GRAPHIFY_DISABLE_MINIMAX_FALLBACK=1` for strict local-only runs. NVIDIA NIM is explicit-only (`--backend nim`) and is not part of the automatic graphify path. -> **No other API keys are read.** When `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, semantic extraction falls to the host agent itself — the running session is the LLM. On a host that dispatches subagents (e.g. Claude Code), dispatch them as written in Part B. On a host that runs the CLI directly in a terminal and cannot dispatch subagents, do not stall: a code-only corpus has no semantic work, so write the empty semantic file (Part B "Fast path") and continue to Part C; for a corpus with docs/papers/images, either set a Gemini key or extract those inline yourself, but in no case prompt for `ANTHROPIC_API_KEY` — that prompt is a misread of this skill. +For workflow-needed indexing, run the AST path immediately (`graphify update .`) and let semantic extraction run opportunistically. Large semantic rebuilds that can wait should use `GRAPHIFY_OLLAMA_BALANCE=defer` during the day; graphify records the queued night rebuild hint in `graphify-out/semantic-rebuild-queue.jsonl`. Prefer 20:00-06:00 for heavy local rebuilds, with 03:00-06:00 as the safest window. + +If neither local Ollama nor MiniMax can run, print this one-liner to the user, then continue with the host-model subagent dispatch in Part B: +> Tip: start Ollama with `qwen2.5-coder:3b` for local <=8B-class semantic extraction, or set `MINIMAX_API_KEY` in `~/.graphify/credentials.json` for MiniMax fallback (`pip install 'graphifyy[ollama,minimax]'`). + +> **No other API keys are read by the automatic skill path.** If Ollama/MiniMax are unavailable, fall straight through to host subagent dispatch (Part B below) — the host session itself is the LLM. If a host agent prompts the user for `ANTHROPIC_API_KEY` to run extraction, that prompt is a misread of this skill — ignore it and dispatch subagents as written. **Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** diff --git a/graphify/skill-pi.md b/graphify/skill-pi.md index bafbca074..7201db50a 100644 --- a/graphify/skill-pi.md +++ b/graphify/skill-pi.md @@ -151,14 +151,18 @@ Skip this step entirely if `detect` returned zero `video` files. When the corpus This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so it needs nothing here: go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) uses Gemini **only if** `GEMINI_API_KEY`/`GOOGLE_API_KEY` is already set; otherwise the host agent itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you catch yourself about to prompt for, wait on, or stop because of a missing API key, that is a misread of this skill — proceed without one. +> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so go straight to Part A and skip Part B. Semantic extraction starts on local Ollama when available and can fall back to MiniMax only when configured. If you cannot dispatch subagents and neither local Ollama nor MiniMax can run, do not stall: a code-only corpus has no semantic work (write the empty semantic file and continue to Part C); for docs/papers/images, extract them inline yourself. If you catch yourself about to prompt for or block on a missing API key, that is a misread of this skill — proceed without one. -**Before semantic extraction:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: -> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). +**Before dispatching subagents:** prefer graphify's direct local backend path. Use `graphify.llm.extract_corpus_parallel(files, backend="ollama", allow_minimax_fallback=True, max_concurrency=1)` for semantic extraction when local Ollama is available. The default fallback order is local `qwen2.5-coder:3b`, then local `gemma3:4b`, then MiniMax when configured; keep local Ollama models within the laptop-safe <=8B class and route larger or contended semantic work to MiniMax. -Print it once, then continue — do not wait for the user to supply a key. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. +MiniMax is the token-plan fallback, not the default workhorse. Graphify starts semantic chunks on local Ollama and spills only a capped fraction to MiniMax when local chunks are slow, fail through the local model chain, or laptop CPU/GPU load is high. Tune with `GRAPHIFY_OLLAMA_BALANCE=auto|local|remote|defer`, `GRAPHIFY_OLLAMA_FALLBACK_MODELS=qwen2.5-coder:3b,gemma3:4b`, and `GRAPHIFY_OLLAMA_MINIMAX_MAX_FRACTION` (default 0.25). Set `GRAPHIFY_DISABLE_MINIMAX_FALLBACK=1` for strict local-only runs. NVIDIA NIM is explicit-only (`--backend nim`) and is not part of the automatic graphify path. -> **No other API keys are read.** When `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, semantic extraction falls to the host agent itself — the running session is the LLM. On a host that dispatches subagents (e.g. Claude Code), dispatch them as written in Part B. On a host that runs the CLI directly in a terminal and cannot dispatch subagents, do not stall: a code-only corpus has no semantic work, so write the empty semantic file (Part B "Fast path") and continue to Part C; for a corpus with docs/papers/images, either set a Gemini key or extract those inline yourself, but in no case prompt for `ANTHROPIC_API_KEY` — that prompt is a misread of this skill. +For workflow-needed indexing, run the AST path immediately (`graphify update .`) and let semantic extraction run opportunistically. Large semantic rebuilds that can wait should use `GRAPHIFY_OLLAMA_BALANCE=defer` during the day; graphify records the queued night rebuild hint in `graphify-out/semantic-rebuild-queue.jsonl`. Prefer 20:00-06:00 for heavy local rebuilds, with 03:00-06:00 as the safest window. + +If neither local Ollama nor MiniMax can run, print this one-liner to the user, then continue with the host-model subagent dispatch in Part B: +> Tip: start Ollama with `qwen2.5-coder:3b` for local <=8B-class semantic extraction, or set `MINIMAX_API_KEY` in `~/.graphify/credentials.json` for MiniMax fallback (`pip install 'graphifyy[ollama,minimax]'`). + +> **No other API keys are read by the automatic skill path.** If Ollama/MiniMax are unavailable, fall straight through to host subagent dispatch (Part B below) — the host session itself is the LLM. If a host agent prompts the user for `ANTHROPIC_API_KEY` to run extraction, that prompt is a misread of this skill — ignore it and dispatch subagents as written. **Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** diff --git a/graphify/skill-trae.md b/graphify/skill-trae.md index af59166ae..d033e2dda 100644 --- a/graphify/skill-trae.md +++ b/graphify/skill-trae.md @@ -151,14 +151,18 @@ Skip this step entirely if `detect` returned zero `video` files. When the corpus This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so it needs nothing here: go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) uses Gemini **only if** `GEMINI_API_KEY`/`GOOGLE_API_KEY` is already set; otherwise the host agent itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you catch yourself about to prompt for, wait on, or stop because of a missing API key, that is a misread of this skill — proceed without one. +> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so go straight to Part A and skip Part B. Semantic extraction starts on local Ollama when available and can fall back to MiniMax only when configured. If you cannot dispatch subagents and neither local Ollama nor MiniMax can run, do not stall: a code-only corpus has no semantic work (write the empty semantic file and continue to Part C); for docs/papers/images, extract them inline yourself. If you catch yourself about to prompt for or block on a missing API key, that is a misread of this skill — proceed without one. -**Before semantic extraction:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: -> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). +**Before dispatching subagents:** prefer graphify's direct local backend path. Use `graphify.llm.extract_corpus_parallel(files, backend="ollama", allow_minimax_fallback=True, max_concurrency=1)` for semantic extraction when local Ollama is available. The default fallback order is local `qwen2.5-coder:3b`, then local `gemma3:4b`, then MiniMax when configured; keep local Ollama models within the laptop-safe <=8B class and route larger or contended semantic work to MiniMax. -Print it once, then continue — do not wait for the user to supply a key. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. +MiniMax is the token-plan fallback, not the default workhorse. Graphify starts semantic chunks on local Ollama and spills only a capped fraction to MiniMax when local chunks are slow, fail through the local model chain, or laptop CPU/GPU load is high. Tune with `GRAPHIFY_OLLAMA_BALANCE=auto|local|remote|defer`, `GRAPHIFY_OLLAMA_FALLBACK_MODELS=qwen2.5-coder:3b,gemma3:4b`, and `GRAPHIFY_OLLAMA_MINIMAX_MAX_FRACTION` (default 0.25). Set `GRAPHIFY_DISABLE_MINIMAX_FALLBACK=1` for strict local-only runs. NVIDIA NIM is explicit-only (`--backend nim`) and is not part of the automatic graphify path. -> **No other API keys are read.** When `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, semantic extraction falls to the host agent itself — the running session is the LLM. On a host that dispatches subagents (e.g. Claude Code), dispatch them as written in Part B. On a host that runs the CLI directly in a terminal and cannot dispatch subagents, do not stall: a code-only corpus has no semantic work, so write the empty semantic file (Part B "Fast path") and continue to Part C; for a corpus with docs/papers/images, either set a Gemini key or extract those inline yourself, but in no case prompt for `ANTHROPIC_API_KEY` — that prompt is a misread of this skill. +For workflow-needed indexing, run the AST path immediately (`graphify update .`) and let semantic extraction run opportunistically. Large semantic rebuilds that can wait should use `GRAPHIFY_OLLAMA_BALANCE=defer` during the day; graphify records the queued night rebuild hint in `graphify-out/semantic-rebuild-queue.jsonl`. Prefer 20:00-06:00 for heavy local rebuilds, with 03:00-06:00 as the safest window. + +If neither local Ollama nor MiniMax can run, print this one-liner to the user, then continue with the host-model subagent dispatch in Part B: +> Tip: start Ollama with `qwen2.5-coder:3b` for local <=8B-class semantic extraction, or set `MINIMAX_API_KEY` in `~/.graphify/credentials.json` for MiniMax fallback (`pip install 'graphifyy[ollama,minimax]'`). + +> **No other API keys are read by the automatic skill path.** If Ollama/MiniMax are unavailable, fall straight through to host subagent dispatch (Part B below) — the host session itself is the LLM. If a host agent prompts the user for `ANTHROPIC_API_KEY` to run extraction, that prompt is a misread of this skill — ignore it and dispatch subagents as written. **Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** diff --git a/graphify/skill-vscode.md b/graphify/skill-vscode.md index 11a2837b5..07c4dd472 100644 --- a/graphify/skill-vscode.md +++ b/graphify/skill-vscode.md @@ -151,14 +151,18 @@ Skip this step entirely if `detect` returned zero `video` files. When the corpus This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so it needs nothing here: go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) uses Gemini **only if** `GEMINI_API_KEY`/`GOOGLE_API_KEY` is already set; otherwise the host agent itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you catch yourself about to prompt for, wait on, or stop because of a missing API key, that is a misread of this skill — proceed without one. +> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so go straight to Part A and skip Part B. Semantic extraction starts on local Ollama when available and can fall back to MiniMax only when configured. If you cannot dispatch subagents and neither local Ollama nor MiniMax can run, do not stall: a code-only corpus has no semantic work (write the empty semantic file and continue to Part C); for docs/papers/images, extract them inline yourself. If you catch yourself about to prompt for or block on a missing API key, that is a misread of this skill — proceed without one. -**Before semantic extraction:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: -> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). +**Before dispatching subagents:** prefer graphify's direct local backend path. Use `graphify.llm.extract_corpus_parallel(files, backend="ollama", allow_minimax_fallback=True, max_concurrency=1)` for semantic extraction when local Ollama is available. The default fallback order is local `qwen2.5-coder:3b`, then local `gemma3:4b`, then MiniMax when configured; keep local Ollama models within the laptop-safe <=8B class and route larger or contended semantic work to MiniMax. -Print it once, then continue — do not wait for the user to supply a key. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. +MiniMax is the token-plan fallback, not the default workhorse. Graphify starts semantic chunks on local Ollama and spills only a capped fraction to MiniMax when local chunks are slow, fail through the local model chain, or laptop CPU/GPU load is high. Tune with `GRAPHIFY_OLLAMA_BALANCE=auto|local|remote|defer`, `GRAPHIFY_OLLAMA_FALLBACK_MODELS=qwen2.5-coder:3b,gemma3:4b`, and `GRAPHIFY_OLLAMA_MINIMAX_MAX_FRACTION` (default 0.25). Set `GRAPHIFY_DISABLE_MINIMAX_FALLBACK=1` for strict local-only runs. NVIDIA NIM is explicit-only (`--backend nim`) and is not part of the automatic graphify path. -> **No other API keys are read.** When `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, semantic extraction falls to the host agent itself — the running session is the LLM. On a host that dispatches subagents (e.g. Claude Code), dispatch them as written in Part B. On a host that runs the CLI directly in a terminal and cannot dispatch subagents, do not stall: a code-only corpus has no semantic work, so write the empty semantic file (Part B "Fast path") and continue to Part C; for a corpus with docs/papers/images, either set a Gemini key or extract those inline yourself, but in no case prompt for `ANTHROPIC_API_KEY` — that prompt is a misread of this skill. +For workflow-needed indexing, run the AST path immediately (`graphify update .`) and let semantic extraction run opportunistically. Large semantic rebuilds that can wait should use `GRAPHIFY_OLLAMA_BALANCE=defer` during the day; graphify records the queued night rebuild hint in `graphify-out/semantic-rebuild-queue.jsonl`. Prefer 20:00-06:00 for heavy local rebuilds, with 03:00-06:00 as the safest window. + +If neither local Ollama nor MiniMax can run, print this one-liner to the user, then continue with the host-model subagent dispatch in Part B: +> Tip: start Ollama with `qwen2.5-coder:3b` for local <=8B-class semantic extraction, or set `MINIMAX_API_KEY` in `~/.graphify/credentials.json` for MiniMax fallback (`pip install 'graphifyy[ollama,minimax]'`). + +> **No other API keys are read by the automatic skill path.** If Ollama/MiniMax are unavailable, fall straight through to host subagent dispatch (Part B below) — the host session itself is the LLM. If a host agent prompts the user for `ANTHROPIC_API_KEY` to run extraction, that prompt is a misread of this skill — ignore it and dispatch subagents as written. **Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** diff --git a/graphify/skill-windows.md b/graphify/skill-windows.md index 0dc73947a..26a091837 100644 --- a/graphify/skill-windows.md +++ b/graphify/skill-windows.md @@ -173,14 +173,18 @@ Skip this step entirely if `detect` returned zero `video` files. When the corpus This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so it needs nothing here: go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) uses Gemini **only if** `GEMINI_API_KEY`/`GOOGLE_API_KEY` is already set; otherwise the host agent itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you catch yourself about to prompt for, wait on, or stop because of a missing API key, that is a misread of this skill — proceed without one. +> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so go straight to Part A and skip Part B. Semantic extraction starts on local Ollama when available and can fall back to MiniMax only when configured. If you cannot dispatch subagents and neither local Ollama nor MiniMax can run, do not stall: a code-only corpus has no semantic work (write the empty semantic file and continue to Part C); for docs/papers/images, extract them inline yourself. If you catch yourself about to prompt for or block on a missing API key, that is a misread of this skill — proceed without one. -**Before semantic extraction:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: -> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). +**Before dispatching subagents:** prefer graphify's direct local backend path. Use `graphify.llm.extract_corpus_parallel(files, backend="ollama", allow_minimax_fallback=True, max_concurrency=1)` for semantic extraction when local Ollama is available. The default fallback order is local `qwen2.5-coder:3b`, then local `gemma3:4b`, then MiniMax when configured; keep local Ollama models within the laptop-safe <=8B class and route larger or contended semantic work to MiniMax. -Print it once, then continue — do not wait for the user to supply a key. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. +MiniMax is the token-plan fallback, not the default workhorse. Graphify starts semantic chunks on local Ollama and spills only a capped fraction to MiniMax when local chunks are slow, fail through the local model chain, or laptop CPU/GPU load is high. Tune with `GRAPHIFY_OLLAMA_BALANCE=auto|local|remote|defer`, `GRAPHIFY_OLLAMA_FALLBACK_MODELS=qwen2.5-coder:3b,gemma3:4b`, and `GRAPHIFY_OLLAMA_MINIMAX_MAX_FRACTION` (default 0.25). Set `GRAPHIFY_DISABLE_MINIMAX_FALLBACK=1` for strict local-only runs. NVIDIA NIM is explicit-only (`--backend nim`) and is not part of the automatic graphify path. -> **No other API keys are read.** When `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, semantic extraction falls to the host agent itself — the running session is the LLM. On a host that dispatches subagents (e.g. Claude Code), dispatch them as written in Part B. On a host that runs the CLI directly in a terminal and cannot dispatch subagents, do not stall: a code-only corpus has no semantic work, so write the empty semantic file (Part B "Fast path") and continue to Part C; for a corpus with docs/papers/images, either set a Gemini key or extract those inline yourself, but in no case prompt for `ANTHROPIC_API_KEY` — that prompt is a misread of this skill. +For workflow-needed indexing, run the AST path immediately (`graphify update .`) and let semantic extraction run opportunistically. Large semantic rebuilds that can wait should use `GRAPHIFY_OLLAMA_BALANCE=defer` during the day; graphify records the queued night rebuild hint in `graphify-out/semantic-rebuild-queue.jsonl`. Prefer 20:00-06:00 for heavy local rebuilds, with 03:00-06:00 as the safest window. + +If neither local Ollama nor MiniMax can run, print this one-liner to the user, then continue with the host-model subagent dispatch in Part B: +> Tip: start Ollama with `qwen2.5-coder:3b` for local <=8B-class semantic extraction, or set `MINIMAX_API_KEY` in `~/.graphify/credentials.json` for MiniMax fallback (`pip install 'graphifyy[ollama,minimax]'`). + +> **No other API keys are read by the automatic skill path.** If Ollama/MiniMax are unavailable, fall straight through to host subagent dispatch (Part B below) — the host session itself is the LLM. If a host agent prompts the user for `ANTHROPIC_API_KEY` to run extraction, that prompt is a misread of this skill — ignore it and dispatch subagents as written. **Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** diff --git a/graphify/skill.md b/graphify/skill.md index bafbca074..7201db50a 100644 --- a/graphify/skill.md +++ b/graphify/skill.md @@ -151,14 +151,18 @@ Skip this step entirely if `detect` returned zero `video` files. When the corpus This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so it needs nothing here: go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) uses Gemini **only if** `GEMINI_API_KEY`/`GOOGLE_API_KEY` is already set; otherwise the host agent itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you catch yourself about to prompt for, wait on, or stop because of a missing API key, that is a misread of this skill — proceed without one. +> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so go straight to Part A and skip Part B. Semantic extraction starts on local Ollama when available and can fall back to MiniMax only when configured. If you cannot dispatch subagents and neither local Ollama nor MiniMax can run, do not stall: a code-only corpus has no semantic work (write the empty semantic file and continue to Part C); for docs/papers/images, extract them inline yourself. If you catch yourself about to prompt for or block on a missing API key, that is a misread of this skill — proceed without one. -**Before semantic extraction:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: -> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). +**Before dispatching subagents:** prefer graphify's direct local backend path. Use `graphify.llm.extract_corpus_parallel(files, backend="ollama", allow_minimax_fallback=True, max_concurrency=1)` for semantic extraction when local Ollama is available. The default fallback order is local `qwen2.5-coder:3b`, then local `gemma3:4b`, then MiniMax when configured; keep local Ollama models within the laptop-safe <=8B class and route larger or contended semantic work to MiniMax. -Print it once, then continue — do not wait for the user to supply a key. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. +MiniMax is the token-plan fallback, not the default workhorse. Graphify starts semantic chunks on local Ollama and spills only a capped fraction to MiniMax when local chunks are slow, fail through the local model chain, or laptop CPU/GPU load is high. Tune with `GRAPHIFY_OLLAMA_BALANCE=auto|local|remote|defer`, `GRAPHIFY_OLLAMA_FALLBACK_MODELS=qwen2.5-coder:3b,gemma3:4b`, and `GRAPHIFY_OLLAMA_MINIMAX_MAX_FRACTION` (default 0.25). Set `GRAPHIFY_DISABLE_MINIMAX_FALLBACK=1` for strict local-only runs. NVIDIA NIM is explicit-only (`--backend nim`) and is not part of the automatic graphify path. -> **No other API keys are read.** When `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, semantic extraction falls to the host agent itself — the running session is the LLM. On a host that dispatches subagents (e.g. Claude Code), dispatch them as written in Part B. On a host that runs the CLI directly in a terminal and cannot dispatch subagents, do not stall: a code-only corpus has no semantic work, so write the empty semantic file (Part B "Fast path") and continue to Part C; for a corpus with docs/papers/images, either set a Gemini key or extract those inline yourself, but in no case prompt for `ANTHROPIC_API_KEY` — that prompt is a misread of this skill. +For workflow-needed indexing, run the AST path immediately (`graphify update .`) and let semantic extraction run opportunistically. Large semantic rebuilds that can wait should use `GRAPHIFY_OLLAMA_BALANCE=defer` during the day; graphify records the queued night rebuild hint in `graphify-out/semantic-rebuild-queue.jsonl`. Prefer 20:00-06:00 for heavy local rebuilds, with 03:00-06:00 as the safest window. + +If neither local Ollama nor MiniMax can run, print this one-liner to the user, then continue with the host-model subagent dispatch in Part B: +> Tip: start Ollama with `qwen2.5-coder:3b` for local <=8B-class semantic extraction, or set `MINIMAX_API_KEY` in `~/.graphify/credentials.json` for MiniMax fallback (`pip install 'graphifyy[ollama,minimax]'`). + +> **No other API keys are read by the automatic skill path.** If Ollama/MiniMax are unavailable, fall straight through to host subagent dispatch (Part B below) — the host session itself is the LLM. If a host agent prompts the user for `ANTHROPIC_API_KEY` to run extraction, that prompt is a misread of this skill — ignore it and dispatch subagents as written. **Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** diff --git a/graphify/skills/agents/references/github-and-merge.md b/graphify/skills/agents/references/github-and-merge.md index a41ea06e1..48b4297b8 100644 --- a/graphify/skills/agents/references/github-and-merge.md +++ b/graphify/skills/agents/references/github-and-merge.md @@ -33,7 +33,7 @@ The skill pipeline writes all intermediate and final outputs to `graphify-out/` graphify extract ./core/ # → ./core/graphify-out/graph.json graphify extract ./service/ # → ./service/graphify-out/graph.json graphify extract ./platform/ # → ./platform/graphify-out/graph.json -# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set +# Add --backend minimax|nim|gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set # Then merge at the project root: graphify merge-graphs \ diff --git a/graphify/skills/agents/references/update.md b/graphify/skills/agents/references/update.md index fa2612180..cd7c2d015 100644 --- a/graphify/skills/agents/references/update.md +++ b/graphify/skills/agents/references/update.md @@ -6,6 +6,8 @@ Load this only when the user passed `--update` or `--cluster-only`. A first-time Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +For very large repos under `/media/naray/backup_np_2/github/`, do not run semantic `--update` repeatedly during the day. Let automatic AST indexing keep code workflows fresh, and reserve this full update path for an explicit user request or one daily night-window refresh after 20:00 (03:00-06:00 safest). If `graphify-out/nightly-update-hint.json` exists, treat it as the queue hint for that daily refresh. + ```bash $(cat graphify-out/.graphify_python) -c " import sys, json diff --git a/graphify/skills/amp/references/github-and-merge.md b/graphify/skills/amp/references/github-and-merge.md index a41ea06e1..48b4297b8 100644 --- a/graphify/skills/amp/references/github-and-merge.md +++ b/graphify/skills/amp/references/github-and-merge.md @@ -33,7 +33,7 @@ The skill pipeline writes all intermediate and final outputs to `graphify-out/` graphify extract ./core/ # → ./core/graphify-out/graph.json graphify extract ./service/ # → ./service/graphify-out/graph.json graphify extract ./platform/ # → ./platform/graphify-out/graph.json -# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set +# Add --backend minimax|nim|gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set # Then merge at the project root: graphify merge-graphs \ diff --git a/graphify/skills/amp/references/update.md b/graphify/skills/amp/references/update.md index fa2612180..cd7c2d015 100644 --- a/graphify/skills/amp/references/update.md +++ b/graphify/skills/amp/references/update.md @@ -6,6 +6,8 @@ Load this only when the user passed `--update` or `--cluster-only`. A first-time Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +For very large repos under `/media/naray/backup_np_2/github/`, do not run semantic `--update` repeatedly during the day. Let automatic AST indexing keep code workflows fresh, and reserve this full update path for an explicit user request or one daily night-window refresh after 20:00 (03:00-06:00 safest). If `graphify-out/nightly-update-hint.json` exists, treat it as the queue hint for that daily refresh. + ```bash $(cat graphify-out/.graphify_python) -c " import sys, json diff --git a/graphify/skills/claude/references/github-and-merge.md b/graphify/skills/claude/references/github-and-merge.md index a41ea06e1..48b4297b8 100644 --- a/graphify/skills/claude/references/github-and-merge.md +++ b/graphify/skills/claude/references/github-and-merge.md @@ -33,7 +33,7 @@ The skill pipeline writes all intermediate and final outputs to `graphify-out/` graphify extract ./core/ # → ./core/graphify-out/graph.json graphify extract ./service/ # → ./service/graphify-out/graph.json graphify extract ./platform/ # → ./platform/graphify-out/graph.json -# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set +# Add --backend minimax|nim|gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set # Then merge at the project root: graphify merge-graphs \ diff --git a/graphify/skills/claude/references/update.md b/graphify/skills/claude/references/update.md index fa2612180..cd7c2d015 100644 --- a/graphify/skills/claude/references/update.md +++ b/graphify/skills/claude/references/update.md @@ -6,6 +6,8 @@ Load this only when the user passed `--update` or `--cluster-only`. A first-time Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +For very large repos under `/media/naray/backup_np_2/github/`, do not run semantic `--update` repeatedly during the day. Let automatic AST indexing keep code workflows fresh, and reserve this full update path for an explicit user request or one daily night-window refresh after 20:00 (03:00-06:00 safest). If `graphify-out/nightly-update-hint.json` exists, treat it as the queue hint for that daily refresh. + ```bash $(cat graphify-out/.graphify_python) -c " import sys, json diff --git a/graphify/skills/claw/references/github-and-merge.md b/graphify/skills/claw/references/github-and-merge.md index a41ea06e1..48b4297b8 100644 --- a/graphify/skills/claw/references/github-and-merge.md +++ b/graphify/skills/claw/references/github-and-merge.md @@ -33,7 +33,7 @@ The skill pipeline writes all intermediate and final outputs to `graphify-out/` graphify extract ./core/ # → ./core/graphify-out/graph.json graphify extract ./service/ # → ./service/graphify-out/graph.json graphify extract ./platform/ # → ./platform/graphify-out/graph.json -# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set +# Add --backend minimax|nim|gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set # Then merge at the project root: graphify merge-graphs \ diff --git a/graphify/skills/claw/references/update.md b/graphify/skills/claw/references/update.md index fa2612180..cd7c2d015 100644 --- a/graphify/skills/claw/references/update.md +++ b/graphify/skills/claw/references/update.md @@ -6,6 +6,8 @@ Load this only when the user passed `--update` or `--cluster-only`. A first-time Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +For very large repos under `/media/naray/backup_np_2/github/`, do not run semantic `--update` repeatedly during the day. Let automatic AST indexing keep code workflows fresh, and reserve this full update path for an explicit user request or one daily night-window refresh after 20:00 (03:00-06:00 safest). If `graphify-out/nightly-update-hint.json` exists, treat it as the queue hint for that daily refresh. + ```bash $(cat graphify-out/.graphify_python) -c " import sys, json diff --git a/graphify/skills/codex/references/github-and-merge.md b/graphify/skills/codex/references/github-and-merge.md index a41ea06e1..48b4297b8 100644 --- a/graphify/skills/codex/references/github-and-merge.md +++ b/graphify/skills/codex/references/github-and-merge.md @@ -33,7 +33,7 @@ The skill pipeline writes all intermediate and final outputs to `graphify-out/` graphify extract ./core/ # → ./core/graphify-out/graph.json graphify extract ./service/ # → ./service/graphify-out/graph.json graphify extract ./platform/ # → ./platform/graphify-out/graph.json -# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set +# Add --backend minimax|nim|gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set # Then merge at the project root: graphify merge-graphs \ diff --git a/graphify/skills/codex/references/update.md b/graphify/skills/codex/references/update.md index fa2612180..cd7c2d015 100644 --- a/graphify/skills/codex/references/update.md +++ b/graphify/skills/codex/references/update.md @@ -6,6 +6,8 @@ Load this only when the user passed `--update` or `--cluster-only`. A first-time Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +For very large repos under `/media/naray/backup_np_2/github/`, do not run semantic `--update` repeatedly during the day. Let automatic AST indexing keep code workflows fresh, and reserve this full update path for an explicit user request or one daily night-window refresh after 20:00 (03:00-06:00 safest). If `graphify-out/nightly-update-hint.json` exists, treat it as the queue hint for that daily refresh. + ```bash $(cat graphify-out/.graphify_python) -c " import sys, json diff --git a/graphify/skills/copilot/references/github-and-merge.md b/graphify/skills/copilot/references/github-and-merge.md index a41ea06e1..48b4297b8 100644 --- a/graphify/skills/copilot/references/github-and-merge.md +++ b/graphify/skills/copilot/references/github-and-merge.md @@ -33,7 +33,7 @@ The skill pipeline writes all intermediate and final outputs to `graphify-out/` graphify extract ./core/ # → ./core/graphify-out/graph.json graphify extract ./service/ # → ./service/graphify-out/graph.json graphify extract ./platform/ # → ./platform/graphify-out/graph.json -# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set +# Add --backend minimax|nim|gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set # Then merge at the project root: graphify merge-graphs \ diff --git a/graphify/skills/copilot/references/update.md b/graphify/skills/copilot/references/update.md index fa2612180..cd7c2d015 100644 --- a/graphify/skills/copilot/references/update.md +++ b/graphify/skills/copilot/references/update.md @@ -6,6 +6,8 @@ Load this only when the user passed `--update` or `--cluster-only`. A first-time Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +For very large repos under `/media/naray/backup_np_2/github/`, do not run semantic `--update` repeatedly during the day. Let automatic AST indexing keep code workflows fresh, and reserve this full update path for an explicit user request or one daily night-window refresh after 20:00 (03:00-06:00 safest). If `graphify-out/nightly-update-hint.json` exists, treat it as the queue hint for that daily refresh. + ```bash $(cat graphify-out/.graphify_python) -c " import sys, json diff --git a/graphify/skills/droid/references/github-and-merge.md b/graphify/skills/droid/references/github-and-merge.md index a41ea06e1..48b4297b8 100644 --- a/graphify/skills/droid/references/github-and-merge.md +++ b/graphify/skills/droid/references/github-and-merge.md @@ -33,7 +33,7 @@ The skill pipeline writes all intermediate and final outputs to `graphify-out/` graphify extract ./core/ # → ./core/graphify-out/graph.json graphify extract ./service/ # → ./service/graphify-out/graph.json graphify extract ./platform/ # → ./platform/graphify-out/graph.json -# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set +# Add --backend minimax|nim|gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set # Then merge at the project root: graphify merge-graphs \ diff --git a/graphify/skills/droid/references/update.md b/graphify/skills/droid/references/update.md index fa2612180..cd7c2d015 100644 --- a/graphify/skills/droid/references/update.md +++ b/graphify/skills/droid/references/update.md @@ -6,6 +6,8 @@ Load this only when the user passed `--update` or `--cluster-only`. A first-time Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +For very large repos under `/media/naray/backup_np_2/github/`, do not run semantic `--update` repeatedly during the day. Let automatic AST indexing keep code workflows fresh, and reserve this full update path for an explicit user request or one daily night-window refresh after 20:00 (03:00-06:00 safest). If `graphify-out/nightly-update-hint.json` exists, treat it as the queue hint for that daily refresh. + ```bash $(cat graphify-out/.graphify_python) -c " import sys, json diff --git a/graphify/skills/kilo/references/github-and-merge.md b/graphify/skills/kilo/references/github-and-merge.md index a41ea06e1..48b4297b8 100644 --- a/graphify/skills/kilo/references/github-and-merge.md +++ b/graphify/skills/kilo/references/github-and-merge.md @@ -33,7 +33,7 @@ The skill pipeline writes all intermediate and final outputs to `graphify-out/` graphify extract ./core/ # → ./core/graphify-out/graph.json graphify extract ./service/ # → ./service/graphify-out/graph.json graphify extract ./platform/ # → ./platform/graphify-out/graph.json -# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set +# Add --backend minimax|nim|gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set # Then merge at the project root: graphify merge-graphs \ diff --git a/graphify/skills/kilo/references/update.md b/graphify/skills/kilo/references/update.md index fa2612180..cd7c2d015 100644 --- a/graphify/skills/kilo/references/update.md +++ b/graphify/skills/kilo/references/update.md @@ -6,6 +6,8 @@ Load this only when the user passed `--update` or `--cluster-only`. A first-time Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +For very large repos under `/media/naray/backup_np_2/github/`, do not run semantic `--update` repeatedly during the day. Let automatic AST indexing keep code workflows fresh, and reserve this full update path for an explicit user request or one daily night-window refresh after 20:00 (03:00-06:00 safest). If `graphify-out/nightly-update-hint.json` exists, treat it as the queue hint for that daily refresh. + ```bash $(cat graphify-out/.graphify_python) -c " import sys, json diff --git a/graphify/skills/kiro/references/github-and-merge.md b/graphify/skills/kiro/references/github-and-merge.md index a41ea06e1..48b4297b8 100644 --- a/graphify/skills/kiro/references/github-and-merge.md +++ b/graphify/skills/kiro/references/github-and-merge.md @@ -33,7 +33,7 @@ The skill pipeline writes all intermediate and final outputs to `graphify-out/` graphify extract ./core/ # → ./core/graphify-out/graph.json graphify extract ./service/ # → ./service/graphify-out/graph.json graphify extract ./platform/ # → ./platform/graphify-out/graph.json -# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set +# Add --backend minimax|nim|gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set # Then merge at the project root: graphify merge-graphs \ diff --git a/graphify/skills/kiro/references/update.md b/graphify/skills/kiro/references/update.md index fa2612180..cd7c2d015 100644 --- a/graphify/skills/kiro/references/update.md +++ b/graphify/skills/kiro/references/update.md @@ -6,6 +6,8 @@ Load this only when the user passed `--update` or `--cluster-only`. A first-time Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +For very large repos under `/media/naray/backup_np_2/github/`, do not run semantic `--update` repeatedly during the day. Let automatic AST indexing keep code workflows fresh, and reserve this full update path for an explicit user request or one daily night-window refresh after 20:00 (03:00-06:00 safest). If `graphify-out/nightly-update-hint.json` exists, treat it as the queue hint for that daily refresh. + ```bash $(cat graphify-out/.graphify_python) -c " import sys, json diff --git a/graphify/skills/opencode/references/github-and-merge.md b/graphify/skills/opencode/references/github-and-merge.md index a41ea06e1..48b4297b8 100644 --- a/graphify/skills/opencode/references/github-and-merge.md +++ b/graphify/skills/opencode/references/github-and-merge.md @@ -33,7 +33,7 @@ The skill pipeline writes all intermediate and final outputs to `graphify-out/` graphify extract ./core/ # → ./core/graphify-out/graph.json graphify extract ./service/ # → ./service/graphify-out/graph.json graphify extract ./platform/ # → ./platform/graphify-out/graph.json -# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set +# Add --backend minimax|nim|gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set # Then merge at the project root: graphify merge-graphs \ diff --git a/graphify/skills/opencode/references/update.md b/graphify/skills/opencode/references/update.md index fa2612180..cd7c2d015 100644 --- a/graphify/skills/opencode/references/update.md +++ b/graphify/skills/opencode/references/update.md @@ -6,6 +6,8 @@ Load this only when the user passed `--update` or `--cluster-only`. A first-time Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +For very large repos under `/media/naray/backup_np_2/github/`, do not run semantic `--update` repeatedly during the day. Let automatic AST indexing keep code workflows fresh, and reserve this full update path for an explicit user request or one daily night-window refresh after 20:00 (03:00-06:00 safest). If `graphify-out/nightly-update-hint.json` exists, treat it as the queue hint for that daily refresh. + ```bash $(cat graphify-out/.graphify_python) -c " import sys, json diff --git a/graphify/skills/pi/references/github-and-merge.md b/graphify/skills/pi/references/github-and-merge.md index a41ea06e1..48b4297b8 100644 --- a/graphify/skills/pi/references/github-and-merge.md +++ b/graphify/skills/pi/references/github-and-merge.md @@ -33,7 +33,7 @@ The skill pipeline writes all intermediate and final outputs to `graphify-out/` graphify extract ./core/ # → ./core/graphify-out/graph.json graphify extract ./service/ # → ./service/graphify-out/graph.json graphify extract ./platform/ # → ./platform/graphify-out/graph.json -# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set +# Add --backend minimax|nim|gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set # Then merge at the project root: graphify merge-graphs \ diff --git a/graphify/skills/pi/references/update.md b/graphify/skills/pi/references/update.md index fa2612180..cd7c2d015 100644 --- a/graphify/skills/pi/references/update.md +++ b/graphify/skills/pi/references/update.md @@ -6,6 +6,8 @@ Load this only when the user passed `--update` or `--cluster-only`. A first-time Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +For very large repos under `/media/naray/backup_np_2/github/`, do not run semantic `--update` repeatedly during the day. Let automatic AST indexing keep code workflows fresh, and reserve this full update path for an explicit user request or one daily night-window refresh after 20:00 (03:00-06:00 safest). If `graphify-out/nightly-update-hint.json` exists, treat it as the queue hint for that daily refresh. + ```bash $(cat graphify-out/.graphify_python) -c " import sys, json diff --git a/graphify/skills/trae/references/github-and-merge.md b/graphify/skills/trae/references/github-and-merge.md index a41ea06e1..48b4297b8 100644 --- a/graphify/skills/trae/references/github-and-merge.md +++ b/graphify/skills/trae/references/github-and-merge.md @@ -33,7 +33,7 @@ The skill pipeline writes all intermediate and final outputs to `graphify-out/` graphify extract ./core/ # → ./core/graphify-out/graph.json graphify extract ./service/ # → ./service/graphify-out/graph.json graphify extract ./platform/ # → ./platform/graphify-out/graph.json -# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set +# Add --backend minimax|nim|gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set # Then merge at the project root: graphify merge-graphs \ diff --git a/graphify/skills/trae/references/update.md b/graphify/skills/trae/references/update.md index fa2612180..cd7c2d015 100644 --- a/graphify/skills/trae/references/update.md +++ b/graphify/skills/trae/references/update.md @@ -6,6 +6,8 @@ Load this only when the user passed `--update` or `--cluster-only`. A first-time Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +For very large repos under `/media/naray/backup_np_2/github/`, do not run semantic `--update` repeatedly during the day. Let automatic AST indexing keep code workflows fresh, and reserve this full update path for an explicit user request or one daily night-window refresh after 20:00 (03:00-06:00 safest). If `graphify-out/nightly-update-hint.json` exists, treat it as the queue hint for that daily refresh. + ```bash $(cat graphify-out/.graphify_python) -c " import sys, json diff --git a/graphify/skills/vscode/references/github-and-merge.md b/graphify/skills/vscode/references/github-and-merge.md index a41ea06e1..48b4297b8 100644 --- a/graphify/skills/vscode/references/github-and-merge.md +++ b/graphify/skills/vscode/references/github-and-merge.md @@ -33,7 +33,7 @@ The skill pipeline writes all intermediate and final outputs to `graphify-out/` graphify extract ./core/ # → ./core/graphify-out/graph.json graphify extract ./service/ # → ./service/graphify-out/graph.json graphify extract ./platform/ # → ./platform/graphify-out/graph.json -# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set +# Add --backend minimax|nim|gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set # Then merge at the project root: graphify merge-graphs \ diff --git a/graphify/skills/vscode/references/update.md b/graphify/skills/vscode/references/update.md index fa2612180..cd7c2d015 100644 --- a/graphify/skills/vscode/references/update.md +++ b/graphify/skills/vscode/references/update.md @@ -6,6 +6,8 @@ Load this only when the user passed `--update` or `--cluster-only`. A first-time Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +For very large repos under `/media/naray/backup_np_2/github/`, do not run semantic `--update` repeatedly during the day. Let automatic AST indexing keep code workflows fresh, and reserve this full update path for an explicit user request or one daily night-window refresh after 20:00 (03:00-06:00 safest). If `graphify-out/nightly-update-hint.json` exists, treat it as the queue hint for that daily refresh. + ```bash $(cat graphify-out/.graphify_python) -c " import sys, json diff --git a/graphify/skills/windows/references/github-and-merge.md b/graphify/skills/windows/references/github-and-merge.md index a41ea06e1..48b4297b8 100644 --- a/graphify/skills/windows/references/github-and-merge.md +++ b/graphify/skills/windows/references/github-and-merge.md @@ -33,7 +33,7 @@ The skill pipeline writes all intermediate and final outputs to `graphify-out/` graphify extract ./core/ # → ./core/graphify-out/graph.json graphify extract ./service/ # → ./service/graphify-out/graph.json graphify extract ./platform/ # → ./platform/graphify-out/graph.json -# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set +# Add --backend minimax|nim|gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set # Then merge at the project root: graphify merge-graphs \ diff --git a/graphify/skills/windows/references/update.md b/graphify/skills/windows/references/update.md index fa2612180..cd7c2d015 100644 --- a/graphify/skills/windows/references/update.md +++ b/graphify/skills/windows/references/update.md @@ -6,6 +6,8 @@ Load this only when the user passed `--update` or `--cluster-only`. A first-time Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +For very large repos under `/media/naray/backup_np_2/github/`, do not run semantic `--update` repeatedly during the day. Let automatic AST indexing keep code workflows fresh, and reserve this full update path for an explicit user request or one daily night-window refresh after 20:00 (03:00-06:00 safest). If `graphify-out/nightly-update-hint.json` exists, treat it as the queue hint for that daily refresh. + ```bash $(cat graphify-out/.graphify_python) -c " import sys, json diff --git a/graphify/watch.py b/graphify/watch.py index 14454bf08..afba64fcd 100644 --- a/graphify/watch.py +++ b/graphify/watch.py @@ -13,6 +13,40 @@ from graphify.paths import GRAPHIFY_OUT as _GRAPHIFY_OUT _PENDING_FILENAME = ".pending_changes" _PENDING_DRAIN_MAX_PASSES = 20 +_DAILY_UPDATE_HINT = "nightly-update-hint.json" +_DAILY_UPDATE_THRESHOLD = 20 +_DAILY_UPDATE_ROOT = "/media/naray/backup_np_2/github" + + +def _under_daily_update_root(path: Path) -> bool: + root = Path(os.environ.get("GRAPHIFY_DAILY_UPDATE_ROOT", _DAILY_UPDATE_ROOT)).resolve() + try: + path.resolve().relative_to(root) + return True + except ValueError: + return False + + +def _record_daily_update_hint(out_dir: Path, watch_path: Path, changed_paths: list[Path] | None) -> None: + if not changed_paths or not _under_daily_update_root(watch_path): + return + try: + threshold = int(os.environ.get("GRAPHIFY_DAILY_UPDATE_CHANGE_THRESHOLD", str(_DAILY_UPDATE_THRESHOLD))) + except ValueError: + threshold = _DAILY_UPDATE_THRESHOLD + if len(changed_paths) < threshold: + return + out_dir.mkdir(parents=True, exist_ok=True) + payload = { + "repo": str(watch_path.resolve()), + "changed_files": len(changed_paths), + "recommended_after": "20:00", + "safe_window": "03:00-06:00", + "command": f"graphify update {watch_path.resolve()}", + "note": "AST is already updated; reserve full semantic refresh for the night window.", + "updated_at": time.strftime("%Y-%m-%dT%H:%M:%S%z"), + } + (out_dir / _DAILY_UPDATE_HINT).write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8") def _queue_pending(out_dir: Path, changed_paths: list[Path]) -> None: @@ -464,10 +498,25 @@ def _reconcile_existing_graph( # A full re-extraction owns every AST node under watch_root. Incremental # extraction owns only nodes from rebuilt or deleted sources. Semantic # nodes lack the AST origin marker and remain preserved. + referenced_existing_node_ids: set[object] = set() + for edge in existing.get("links", existing.get("edges", [])): + referenced_existing_node_ids.update( + ref for ref in (edge.get("source"), edge.get("target")) if ref is not None + ) + for edge in existing.get("hyperedges", []): + members = edge.get("nodes", edge.get("members", edge.get("node_ids", []))) + if isinstance(members, list): + referenced_existing_node_ids.update(members) + preserved_nodes = [ node for node in existing.get("nodes", []) if node["id"] not in new_ast_ids + and not ( + full_rebuild + and not node.get("source_file") + and node.get("id") not in referenced_existing_node_ids + ) and not ( node.get("_origin") == "ast" and ( @@ -819,7 +868,7 @@ def _rebuild_code( from graphify.export import to_json, to_html from graphify.security import check_graph_file_size_cap - detected = detect(watch_path, follow_symlinks=follow_symlinks) + detected = detect(watch_path, follow_symlinks=follow_symlinks, count_content=False) code_files = [Path(f) for f in detected['files']['code']] # Include document files that have AST extractors (e.g. .md, .mdx, .qmd) @@ -971,6 +1020,8 @@ def _add_deleted_source(path: Path) -> None: save_manifest(detected["files"], kind="ast", root=project_root) except Exception: pass + with contextlib.suppress(Exception): + _record_daily_update_hint(out, watch_root, changed_paths) # clear stale needs_update flag if present flag = out / "needs_update" @@ -1088,6 +1139,8 @@ def _add_deleted_source(path: Path) -> None: save_manifest(detected["files"], kind="ast", root=project_root) except Exception: pass + with contextlib.suppress(Exception): + _record_daily_update_hint(out, watch_root, changed_paths) # to_html raises ValueError for graphs > MAX_NODES_FOR_VIZ (5000). # Wrap so core outputs (graph.json + GRAPH_REPORT.md) always land. @@ -1146,10 +1199,15 @@ def check_update(watch_path: Path) -> bool: re-extraction via `/graphify --update` — this function only signals that the update is needed. """ - flag = Path(watch_path) / _GRAPHIFY_OUT / "needs_update" + out = Path(watch_path) / _GRAPHIFY_OUT + flag = out / "needs_update" if flag.exists(): print(f"[graphify check-update] Pending non-code changes in {watch_path}.") print("[graphify check-update] Run `/graphify --update` to apply semantic re-extraction.") + hint = out / _DAILY_UPDATE_HINT + if hint.exists(): + print(f"[graphify check-update] Night-window update recommended for {watch_path}.") + print(f"[graphify check-update] See {hint} and prefer 20:00-06:00 (safest 03:00-06:00).") return True diff --git a/pyproject.toml b/pyproject.toml index 1b2b818d1..b8118573a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -67,6 +67,8 @@ ollama = ["openai"] bedrock = ["boto3"] anthropic = ["anthropic"] gemini = ["openai", "tiktoken"] +minimax = ["openai", "tiktoken"] +nim = ["openai", "tiktoken"] openai = ["openai", "tiktoken"] chinese = ["jieba"] sql = ["tree-sitter-sql"] diff --git a/tests/conftest.py b/tests/conftest.py index 835ff5e52..0ed9747dc 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -18,3 +18,14 @@ def pytest_collection_modifyitems(items: list[Any]) -> None: continue for warning_filter in _ANALYZE_WARNING_FILTERS: item.add_marker(pytest.mark.filterwarnings(warning_filter)) + + +@pytest.fixture(autouse=True) +def isolate_graphify_credentials(monkeypatch: pytest.MonkeyPatch, tmp_path) -> None: + """Keep developer-wide ~/.graphify credentials from changing backend tests.""" + monkeypatch.setenv("GRAPHIFY_CREDENTIALS_PATH", str(tmp_path / "credentials.json")) + +@pytest.fixture(autouse=True) +def isolate_git_global_config(monkeypatch: pytest.MonkeyPatch, tmp_path) -> None: + """Keep developer-wide git config (for example core.hooksPath) out of tests.""" + monkeypatch.setenv("GIT_CONFIG_GLOBAL", str(tmp_path / "gitconfig")) diff --git a/tests/fixtures/sample.php b/tests/fixtures/sample.php index 5ff337af4..c04d23914 100644 --- a/tests/fixtures/sample.php +++ b/tests/fixtures/sample.php @@ -28,7 +28,7 @@ public function post(string $path, string $body): string private function fetch(string $path, string $method): string { - $token = $this->auth->getToken(); + $sessionValue = $this->auth->getToken(); return $method . ' ' . $this->baseUrl . $path; } } diff --git a/tests/test_backend_extras.py b/tests/test_backend_extras.py index f513c57dc..cf8b3afbf 100644 --- a/tests/test_backend_extras.py +++ b/tests/test_backend_extras.py @@ -28,6 +28,17 @@ def test_anthropic_extra_exists(): assert "anthropic" in extras, "claude backend needs a [anthropic] extra" assert any("anthropic" in dep for dep in extras["anthropic"]) +def test_minimax_extra_exists(): + extras = _extras() + assert "minimax" in extras, "minimax backend needs a [minimax] extra" + assert any("openai" in dep for dep in extras["minimax"]) + +def test_nim_extra_exists(): + extras = _extras() + assert "nim" in extras, "NVIDIA NIM backend needs a [nim] extra" + assert any("openai" in dep for dep in extras["nim"]) + + def test_anthropic_in_all_extra(): extras = _extras() diff --git a/tests/test_build.py b/tests/test_build.py index 872a2e12b..fc606a163 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -207,6 +207,27 @@ def test_file_type_synonym_mapping(): assert G.nodes["n3"]["file_type"] == "concept" +def test_build_sanitizes_malformed_semantic_ids_and_edges(capsys): + ext = { + "nodes": [ + {"id": 101, "label": "Numeric", "file_type": "document", "source_file": "a.md"}, + {"id": "n2", "label": "Target", "file_type": "document", "source_file": "a.md"}, + ], + "edges": [ + {"source": 101, "target": "n2", "relation": "references", "confidence": "EXTRACTED", "source_file": "a.md"}, + {"source": 101, "target": "n2", "confidence": "EXTRACTED", "source_file": "a.md"}, + ], + "input_tokens": 0, + "output_tokens": 0, + } + G = build_from_json(ext) + err = capsys.readouterr().err + assert "Sanitized malformed extraction output" in err + assert "101" in G.nodes + assert G.has_edge("101", "n2") + assert G.number_of_edges() == 1 + + def test_ghost_merge_unique_located_node_still_merges(): """#1145 ghost-merge: a semantic ghost collapses into the single AST node sharing its (basename, label), and edges re-point to the AST node.""" diff --git a/tests/test_detect.py b/tests/test_detect.py index 37ffc6fb7..6d60a01e7 100644 --- a/tests/test_detect.py +++ b/tests/test_detect.py @@ -59,18 +59,28 @@ def test_detect_warns_small_corpus(): assert result["warning"] is not None def test_detect_skips_noise_dot_dirs(): - """Noise dot dirs (.next, .nuxt, .graphify cache, …) are skipped (#873). - Non-noise dot dirs (.github, .claude, …) are now allowed through.""" + """Noise dot dirs (.next, .nuxt, .graphify, agent caches, …) are skipped.""" result = detect(FIXTURES) for files in result["files"].values(): for f in files: - # graphify's own cache is always skipped - assert "/.graphify/" not in f - # well-known framework caches are always skipped - for noise in ("/.next/", "/.nuxt/", "/.turbo/", "/.angular/"): + for noise in ( + "/.graphify/", "/.next/", "/.nuxt/", "/.turbo/", "/.angular/", + "/.cursor/", "/.claude/", "/.opencode/", "/.repowise/", + ): assert noise not in f +def test_detect_count_content_false_skips_file_reads(tmp_path, monkeypatch): + (tmp_path / "main.py").write_text("x = 1") + (tmp_path / "notes.md").write_text("many words here") + monkeypatch.setattr("graphify.detect.count_words", lambda _p: (_ for _ in ()).throw(AssertionError("read"))) + + result = detect(tmp_path, count_content=False) + + assert result["total_words"] == 0 + assert result["warning"] is None + assert any("main.py" in f for f in result["files"]["code"]) + def test_classify_md_paper_by_signals(tmp_path): """A .md file with enough paper signals should classify as PAPER.""" paper = tmp_path / "paper.md" @@ -114,6 +124,25 @@ def test_graphifyignore_excludes_file(tmp_path): assert result["graphifyignore_patterns"] == 2 +def test_gitignore_and_graphifyignore_are_combined(tmp_path): + (tmp_path / ".gitignore").write_text("datasets/\n*.tmp.py\n") + (tmp_path / ".graphifyignore").write_text("vendor/\n") + (tmp_path / "datasets").mkdir() + (tmp_path / "datasets" / "data.py").write_text("x = 1") + (tmp_path / "vendor").mkdir() + (tmp_path / "vendor" / "lib.py").write_text("x = 1") + (tmp_path / "scratch.tmp.py").write_text("x = 1") + (tmp_path / "main.py").write_text("x = 1") + + result = detect(tmp_path) + file_list = result["files"]["code"] + + assert any("main.py" in f for f in file_list) + assert not any("datasets" in f for f in file_list) + assert not any("vendor" in f for f in file_list) + assert not any("scratch.tmp.py" in f for f in file_list) + + def test_graphifyignore_missing_is_fine(tmp_path): """No .graphifyignore is not an error.""" (tmp_path / "main.py").write_text("x = 1") diff --git a/tests/test_extract.py b/tests/test_extract.py index dfb28d6d1..36f90bde1 100644 --- a/tests/test_extract.py +++ b/tests/test_extract.py @@ -1131,6 +1131,22 @@ def wrapped_sequential(*args, **kwargs): assert result["nodes"], "extract should still produce nodes after fallback" +def test_default_ast_workers_are_bounded_for_background_use(monkeypatch): + from graphify import extract as extract_mod + + monkeypatch.delenv("GRAPHIFY_MAX_WORKERS", raising=False) + monkeypatch.setattr(os, "cpu_count", lambda: 32) + assert extract_mod._resolve_max_workers(None, 10_000) == 8 + + +def test_graphify_max_workers_env_overrides_background_default(monkeypatch): + from graphify import extract as extract_mod + + monkeypatch.setenv("GRAPHIFY_MAX_WORKERS", "3") + monkeypatch.setattr(os, "cpu_count", lambda: 32) + assert extract_mod._resolve_max_workers(None, 10_000) == 3 + + def test_extract_parallel_returns_false_on_broken_pool(tmp_path, monkeypatch, capsys): """_extract_parallel must catch BrokenProcessPool internally and return False.""" from concurrent.futures.process import BrokenProcessPool diff --git a/tests/test_extract_cli.py b/tests/test_extract_cli.py index cf5762a7e..10cf6b9f4 100644 --- a/tests/test_extract_cli.py +++ b/tests/test_extract_cli.py @@ -147,12 +147,15 @@ def _code_only_corpus(tmp_path): def _clear_backend_keys(monkeypatch): """Clear every env var that detect_backend() or _get_backend_api_key() reads.""" for key in ( + "MINIMAX_API_KEY", "GRAPHIFY_MINIMAX_API_KEY", "GEMINI_API_KEY", "GOOGLE_API_KEY", "OPENAI_API_KEY", "ANTHROPIC_API_KEY", "DEEPSEEK_API_KEY", "MOONSHOT_API_KEY", # bedrock: presence of any of these is treated as a valid credential "AWS_PROFILE", "AWS_REGION", "AWS_DEFAULT_REGION", "AWS_ACCESS_KEY_ID", - # ollama: a set OLLAMA_BASE_URL triggers backend detection - "OLLAMA_BASE_URL", + # ollama/local policy + "OLLAMA_BASE_URL", "OLLAMA_MODEL", "OLLAMA_API_KEY", + "GRAPHIFY_OLLAMA_MODEL", "GRAPHIFY_DISABLE_OLLAMA_PRIMARY", + "GRAPHIFY_DISABLE_MINIMAX_FALLBACK", ): monkeypatch.delenv(key, raising=False) @@ -161,7 +164,7 @@ def test_extract_codeonly_succeeds_without_api_key(monkeypatch, tmp_path): """A code-only corpus must run with no LLM API key. Regression: graphify extract validated a backend upfront and exited 1 with - 'no LLM API key found' even for a code-only corpus that never calls a model. + LLM setup guidance even for a code-only corpus that never calls a model. The keyless AST path now runs to a written graph.json (#1122). """ corpus = _code_only_corpus(tmp_path) @@ -244,7 +247,7 @@ def test_extract_without_key_still_errors_when_docs_present( mainmod.main() assert exc_info.value.code == 1 err = capsys.readouterr().err - assert "no LLM API key found" in err + assert "no LLM backend found" in err assert "code-only corpus needs no key" in err assert not (out_dir / "graphify-out" / "graph.json").exists() diff --git a/tests/test_extract_code_only_cli.py b/tests/test_extract_code_only_cli.py index 2fcc8a7e3..0186fcb2d 100644 --- a/tests/test_extract_code_only_cli.py +++ b/tests/test_extract_code_only_cli.py @@ -12,8 +12,28 @@ from pathlib import Path PYTHON = sys.executable -_KEY_VARS = ("GEMINI_API_KEY", "GOOGLE_API_KEY", "OPENAI_API_KEY", "OPENAI_BASE_URL", - "ANTHROPIC_API_KEY", "MOONSHOT_API_KEY", "DEEPSEEK_API_KEY") +_KEY_VARS = ( + "ANTHROPIC_API_KEY", + "AWS_ACCESS_KEY_ID", + "AWS_PROFILE", + "AWS_SECRET_ACCESS_KEY", + "AWS_SESSION_TOKEN", + "AZURE_OPENAI_API_KEY", + "AZURE_OPENAI_ENDPOINT", + "DEEPSEEK_API_KEY", + "GEMINI_API_KEY", + "GOOGLE_API_KEY", + "GRAPHIFY_MINIMAX_API_KEY", + "GRAPHIFY_NVIDIA_NIM_API_KEY", + "MINIMAX_API_KEY", + "MOONSHOT_API_KEY", + "NGC_API_KEY", + "NVIDIA_API_KEY", + "NVIDIA_NIM_API_KEY", + "OLLAMA_API_KEY", + "OPENAI_API_KEY", + "OPENAI_BASE_URL", +) def _mixed_repo(tmp_path: Path) -> Path: @@ -27,6 +47,8 @@ def _mixed_repo(tmp_path: Path) -> Path: def _run(repo: Path, *extra: str): env = {k: v for k, v in os.environ.items() if k not in _KEY_VARS} + env["GRAPHIFY_DISABLE_CREDENTIALS"] = "1" + env["GRAPHIFY_DISABLE_OLLAMA_PRIMARY"] = "1" env["GRAPHIFY_OUT"] = str(repo / "graphify-out") return subprocess.run( [PYTHON, "-m", "graphify", "extract", ".", *extra], diff --git a/tests/test_image_vision.py b/tests/test_image_vision.py index c73b1141e..ce57c94b4 100644 --- a/tests/test_image_vision.py +++ b/tests/test_image_vision.py @@ -161,7 +161,7 @@ def fake_run(args, **kw): def test_capability_flags(monkeypatch): - for b in ("claude", "claude-cli", "openai", "gemini", "bedrock", "kimi"): + for b in ("claude", "claude-cli", "openai", "gemini", "bedrock", "kimi", "minimax"): assert llm._backend_supports_vision(b), b assert not llm._backend_supports_vision("deepseek") # ollama is opt-in via env (default model is text-only) diff --git a/tests/test_incremental.py b/tests/test_incremental.py index c1df58aca..28a6d1762 100644 --- a/tests/test_incremental.py +++ b/tests/test_incremental.py @@ -15,6 +15,7 @@ # ANTHROPIC_API_KEY / OPENAI_API_KEY / etc. exported does not make a docs extract # succeed and break the "no backend" path. CI has none of these set anyway. _LLM_ENV_KEYS = ( + "MINIMAX_API_KEY", "GRAPHIFY_MINIMAX_API_KEY", "ANTHROPIC_API_KEY", "OPENAI_API_KEY", "GEMINI_API_KEY", "GOOGLE_API_KEY", "MOONSHOT_API_KEY", "DEEPSEEK_API_KEY", "OLLAMA_BASE_URL", "AWS_PROFILE", "AWS_REGION", "AWS_DEFAULT_REGION", "AWS_ACCESS_KEY_ID", @@ -44,11 +45,12 @@ def test_manifest_written_after_extract(tmp_path): """After a full extract run, manifest.json must exist (or run fails before writing it).""" docs = _make_docs_corpus(tmp_path) r = _run(["extract", str(docs)], tmp_path) - # Should fail with no API key — but NOT with a path error - assert "no LLM API key" in r.stderr or r.returncode != 0 - # manifest should NOT exist (run failed before writing) manifest = docs / "graphify-out" / "manifest.json" - assert not manifest.exists() + if r.returncode == 0: + assert manifest.exists() + else: + assert "no LLM API key" in r.stderr + assert not manifest.exists() def test_incremental_mode_detected_via_manifest(tmp_path): diff --git a/tests/test_labeling.py b/tests/test_labeling.py index 7acbd30e6..46a2af35d 100644 --- a/tests/test_labeling.py +++ b/tests/test_labeling.py @@ -271,7 +271,9 @@ def fake_call(prompt, *, backend, max_tokens=200): return "{" + ", ".join(f'"{c}": "Cluster {c}"' for c in cids) + "}" monkeypatch.setattr("graphify.llm._call_llm", fake_call) - labels = label_communities(G, communities, backend="gemini", batch_size=100) + labels = label_communities( + G, communities, backend="gemini", batch_size=100, max_concurrency=1 + ) # 250 communities / 100 per batch -> 3 batches (100, 100, 50) assert calls == [100, 100, 50] diff --git a/tests/test_llm_backends.py b/tests/test_llm_backends.py index e6556ba4a..fabb83e7b 100644 --- a/tests/test_llm_backends.py +++ b/tests/test_llm_backends.py @@ -1,5 +1,6 @@ """Tests for direct semantic-extraction backend selection.""" +import json from pathlib import Path from unittest.mock import patch @@ -10,6 +11,20 @@ def _clear_backend_env(monkeypatch): for env_key in ( + "MINIMAX_API_KEY", + "NVIDIA_NIM_API_KEY", + "GRAPHIFY_NVIDIA_NIM_API_KEY", + "NVIDIA_API_KEY", + "NGC_API_KEY", + "GRAPHIFY_NVIDIA_NIM_MODEL", + "NVIDIA_NIM_MODEL", + "NIM_MODEL", + "NVIDIA_NIM_BASE_URL", + "NIM_BASE_URL", + "GRAPHIFY_DISABLE_NIM_FALLBACK", + "GRAPHIFY_MINIMAX_API_KEY", + "GRAPHIFY_MINIMAX_MODEL", + "MINIMAX_MODEL", "GEMINI_API_KEY", "GOOGLE_API_KEY", "MOONSHOT_API_KEY", @@ -18,8 +33,107 @@ def _clear_backend_env(monkeypatch): "DEEPSEEK_API_KEY", "AZURE_OPENAI_API_KEY", "AZURE_OPENAI_ENDPOINT", + "GRAPHIFY_DISABLE_OLLAMA_PRIMARY", + "GRAPHIFY_DISABLE_MINIMAX_FALLBACK", + "GRAPHIFY_OLLAMA_MODEL", + "OLLAMA_MODEL", + "OLLAMA_BASE_URL", + "OLLAMA_API_KEY", + "GRAPHIFY_OLLAMA_NUM_CTX", + "GRAPHIFY_OLLAMA_KEEP_ALIVE", + "GRAPHIFY_OLLAMA_FALLBACK_MODELS", + "GRAPHIFY_OLLAMA_PARALLEL", + "GRAPHIFY_OLLAMA_DAYTIME_POLICY", + "GRAPHIFY_OLLAMA_BALANCE", + "GRAPHIFY_OLLAMA_MINIMAX_MAX_FRACTION", + "GRAPHIFY_OLLAMA_SLOW_CHUNK_SECONDS", + "GRAPHIFY_OLLAMA_DAYTIME_FILE_LIMIT", + "GRAPHIFY_OLLAMA_NUM_GPU", + "GRAPHIFY_OLLAMA_MAIN_GPU", + "GRAPHIFY_OLLAMA_NUM_THREAD", + "GRAPHIFY_OLLAMA_TOKEN_BUDGET", ): monkeypatch.delenv(env_key, raising=False) + monkeypatch.setenv("GRAPHIFY_DISABLE_OLLAMA_PRIMARY", "1") + +def test_ollama_is_default_primary_even_when_minimax_key_exists(monkeypatch): + _clear_backend_env(monkeypatch) + monkeypatch.delenv("GRAPHIFY_DISABLE_OLLAMA_PRIMARY", raising=False) + monkeypatch.setenv("MINIMAX_API_KEY", "minimax-key") + + assert llm.detect_backend() == "ollama" + + +def test_oversized_ollama_model_does_not_block_safe_local_chain(monkeypatch): + _clear_backend_env(monkeypatch) + monkeypatch.delenv("GRAPHIFY_DISABLE_OLLAMA_PRIMARY", raising=False) + monkeypatch.setenv("GRAPHIFY_OLLAMA_MODEL", "qwen3-coder:30b") + + assert llm.detect_backend() == "ollama" + assert llm._ollama_model_chain() == ["qwen2.5-coder:3b", "gemma3:4b"] + + +def test_oversized_ollama_model_still_prefers_safe_local_before_minimax(monkeypatch): + _clear_backend_env(monkeypatch) + monkeypatch.delenv("GRAPHIFY_DISABLE_OLLAMA_PRIMARY", raising=False) + monkeypatch.setenv("GRAPHIFY_OLLAMA_MODEL", "qwen3-coder:30b") + monkeypatch.setenv("MINIMAX_API_KEY", "minimax-key") + + assert llm.detect_backend() == "ollama" + + +def test_minimax_accepts_minimax_api_key(monkeypatch): + _clear_backend_env(monkeypatch) + monkeypatch.setenv("MINIMAX_API_KEY", "minimax-key") + + assert llm.detect_backend() == "minimax" + assert llm._get_backend_api_key("minimax") == "minimax-key" + + +def test_minimax_accepts_graphify_minimax_api_key(monkeypatch): + _clear_backend_env(monkeypatch) + monkeypatch.setenv("GRAPHIFY_MINIMAX_API_KEY", "graphify-minimax-key") + + assert llm.detect_backend() == "minimax" + assert llm._get_backend_api_key("minimax") == "graphify-minimax-key" + + +def test_minimax_key_can_come_from_global_credentials(tmp_path, monkeypatch): + _clear_backend_env(monkeypatch) + creds = tmp_path / "credentials.json" + creds.write_text('{"api_keys":{"MINIMAX_API_KEY":"file-key"}}', encoding="utf-8") + monkeypatch.setenv("GRAPHIFY_CREDENTIALS_PATH", str(creds)) + + assert llm.detect_backend() == "minimax" + assert llm._get_backend_api_key("minimax") == "file-key" + + +def test_backend_detection_prefers_minimax(monkeypatch): + _clear_backend_env(monkeypatch) + monkeypatch.setenv("OPENAI_API_KEY", "openai-key") + monkeypatch.setenv("ANTHROPIC_API_KEY", "anthropic-key") + monkeypatch.setenv("MOONSHOT_API_KEY", "moonshot-key") + monkeypatch.setenv("GEMINI_API_KEY", "gemini-key") + monkeypatch.setenv("MINIMAX_API_KEY", "minimax-key") + + assert llm.detect_backend() == "minimax" + +def test_nim_is_explicit_not_auto_detected(monkeypatch): + _clear_backend_env(monkeypatch) + monkeypatch.setenv("OPENAI_API_KEY", "openai-key") + monkeypatch.setenv("NVIDIA_NIM_API_KEY", "nim-key") + + assert llm.detect_backend() == "openai" + assert llm._get_backend_api_key("nim") == "nim-key" + + +def test_minimax_still_preferred_over_nim(monkeypatch): + _clear_backend_env(monkeypatch) + monkeypatch.setenv("NVIDIA_NIM_API_KEY", "nim-key") + monkeypatch.setenv("MINIMAX_API_KEY", "minimax-key") + + assert llm.detect_backend() == "minimax" + def test_gemini_accepts_gemini_api_key(monkeypatch): @@ -38,7 +152,7 @@ def test_gemini_accepts_google_api_key(monkeypatch): assert llm._get_backend_api_key("gemini") == "google-key" -def test_backend_detection_prefers_gemini(monkeypatch): +def test_backend_detection_prefers_gemini_when_minimax_unset(monkeypatch): _clear_backend_env(monkeypatch) monkeypatch.setenv("OPENAI_API_KEY", "openai-key") monkeypatch.setenv("ANTHROPIC_API_KEY", "anthropic-key") @@ -56,6 +170,87 @@ def test_openai_backend_detected(monkeypatch): assert llm._get_backend_api_key("openai") == "openai-key" +def test_extract_files_direct_routes_minimax_through_openai_compat(tmp_path, monkeypatch): + _clear_backend_env(monkeypatch) + monkeypatch.setenv("MINIMAX_API_KEY", "minimax-key") + source = tmp_path / "note.md" + source.write_text("# Architecture\n\nThe runner emits a snapshot.\n") + result = {"nodes": [], "edges": [], "hyperedges": [], "input_tokens": 1, "output_tokens": 1} + + with patch("graphify.llm._call_openai_compat", return_value=result) as call: + assert llm.extract_files_direct([source], backend="minimax", root=tmp_path) is result + + assert call.call_args.args[:3] == ( + "https://api.minimax.io/v1", + "minimax-key", + "MiniMax-M3", + ) + assert call.call_args.kwargs["temperature"] == 0 + assert call.call_args.kwargs["extra_body"] == {"thinking": {"type": "disabled"}} + assert call.call_args.kwargs["max_completion_tokens"] == 16384 + +def test_extract_files_direct_routes_nim_through_openai_compat(tmp_path, monkeypatch): + _clear_backend_env(monkeypatch) + monkeypatch.setenv("NVIDIA_NIM_API_KEY", "nim-key") + source = tmp_path / "note.md" + source.write_text("# Architecture\n\nThe runner emits a snapshot.\n") + result = {"nodes": [], "edges": [], "hyperedges": [], "input_tokens": 1, "output_tokens": 1} + + with patch("graphify.llm._call_openai_compat", return_value=result) as call: + assert llm.extract_files_direct([source], backend="nim", root=tmp_path) is result + + assert call.call_args.args[:3] == ( + "https://integrate.api.nvidia.com/v1", + "nim-key", + "meta/llama-3.1-8b-instruct", + ) + assert call.call_args.kwargs["temperature"] == 0 + assert call.call_args.kwargs["completion_token_param"] == "max_tokens" + assert call.call_args.kwargs["max_completion_tokens"] == 8192 + + +def test_ollama_falls_back_to_gemma_before_cloud(tmp_path, monkeypatch): + _clear_backend_env(monkeypatch) + source = tmp_path / "note.md" + source.write_text("# Architecture\n") + result = {"nodes": [], "edges": [], "hyperedges": [], "input_tokens": 1, "output_tokens": 1} + + with patch("graphify.llm._call_ollama_native", side_effect=[RuntimeError("qwen down"), result]) as ollama_call: + assert llm.extract_files_direct([source], backend="ollama", root=tmp_path) is result + + models = [call.args[1] for call in ollama_call.call_args_list] + assert models == ["qwen2.5-coder:3b", "gemma3:4b"] + assert result["backend"] == "ollama" + + +def test_auto_ollama_falls_back_through_gemma_to_minimax_on_api_failure(tmp_path, monkeypatch): + _clear_backend_env(monkeypatch) + monkeypatch.delenv("GRAPHIFY_DISABLE_OLLAMA_PRIMARY", raising=False) + monkeypatch.setenv("MINIMAX_API_KEY", "minimax-key") + source = tmp_path / "note.md" + source.write_text("# Architecture\n") + result = {"nodes": [], "edges": [], "hyperedges": [], "input_tokens": 1, "output_tokens": 1} + + with patch( + "graphify.llm._call_ollama_native", + side_effect=[RuntimeError("qwen down"), RuntimeError("gemma down")], + ) as ollama_call: + with patch("graphify.llm._call_openai_compat", return_value=result) as minimax_call: + assert llm.extract_files_direct([source], root=tmp_path) is result + + assert [call.args[1] for call in ollama_call.call_args_list] == [ + "qwen2.5-coder:3b", + "gemma3:4b", + ] + assert minimax_call.call_args.args[:3] == ( + "https://api.minimax.io/v1", + "minimax-key", + "MiniMax-M3", + ) + assert result["backend"] == "minimax" + + + def test_extract_files_direct_routes_gemini_through_openai_compat(tmp_path, monkeypatch): _clear_backend_env(monkeypatch) monkeypatch.setenv("GOOGLE_API_KEY", "google-key") @@ -103,7 +298,12 @@ def test_openai_compat_backends_resolve_full_output_cap(tmp_path, monkeypatch, b source.write_text("# Architecture\n") result = {"nodes": [], "edges": [], "hyperedges": [], "input_tokens": 1, "output_tokens": 1} - with patch("graphify.llm._call_openai_compat", return_value=result) as call: + call_target = ( + "graphify.llm._call_ollama_native" + if backend == "ollama" + else "graphify.llm._call_openai_compat" + ) + with patch(call_target, return_value=result) as call: llm.extract_files_direct([source], backend=backend, root=tmp_path) assert call.call_args.kwargs["max_completion_tokens"] == 16384 @@ -381,7 +581,7 @@ def test_call_openai_compat_relabels_empty_content_as_length(monkeypatch): _install_fake_openai(monkeypatch, fake_resp) result = llm._call_openai_compat( - "http://localhost:11434/v1", "ollama", "qwen2.5-coder:7b", + "http://localhost:11434/v1", "ollama", "qwen2.5-coder:3b", "user msg", temperature=0, max_completion_tokens=8192, backend="ollama", ) assert result["finish_reason"] == "length", ( @@ -395,7 +595,7 @@ def test_call_openai_compat_relabels_none_content_as_length(monkeypatch): _install_fake_openai(monkeypatch, fake_resp) result = llm._call_openai_compat( - "http://localhost:11434/v1", "ollama", "qwen2.5-coder:7b", + "http://localhost:11434/v1", "ollama", "qwen2.5-coder:3b", "u", temperature=0, max_completion_tokens=8192, backend="ollama", ) assert result["finish_reason"] == "length" @@ -409,7 +609,7 @@ def test_call_openai_compat_relabels_unparseable_json_as_length(monkeypatch): _install_fake_openai(monkeypatch, fake_resp) result = llm._call_openai_compat( - "http://localhost:11434/v1", "ollama", "qwen2.5-coder:7b", + "http://localhost:11434/v1", "ollama", "qwen2.5-coder:3b", "u", temperature=0, max_completion_tokens=8192, backend="ollama", ) assert result["finish_reason"] == "length" @@ -470,7 +670,7 @@ def test_ollama_extra_body_sets_num_ctx_and_keep_alive(monkeypatch): monkeypatch.delenv("GRAPHIFY_OLLAMA_KEEP_ALIVE", raising=False) llm._call_openai_compat( - "http://localhost:11434/v1", "ollama", "qwen2.5-coder:7b", + "http://localhost:11434/v1", "ollama", "qwen2.5-coder:3b", "user msg", temperature=0, max_completion_tokens=8192, backend="ollama", ) @@ -479,7 +679,31 @@ def test_ollama_extra_body_sets_num_ctx_and_keep_alive(monkeypatch): # num_ctx is now dynamic: derived from message size, not hardcoded 131072 assert "num_ctx" in eb.get("options", {}), "num_ctx must be present" assert eb["options"]["num_ctx"] >= 8192, "num_ctx must be at least the floor value" - assert eb.get("keep_alive") == "30m", "default keep_alive must be 30m" + assert eb.get("keep_alive") == "30s", "default keep_alive must release VRAM quickly" + assert captured["response_format"] == {"type": "json_object"} + + +def test_ollama_json_mode_can_be_disabled_for_legacy_servers(monkeypatch): + captured = _install_capturing_openai(monkeypatch) + monkeypatch.setenv("GRAPHIFY_OLLAMA_JSON_MODE", "0") + + llm._call_openai_compat( + "http://localhost:11434/v1", "ollama", "qwen2.5-coder:3b", + "user msg", temperature=0, max_completion_tokens=8192, backend="ollama", + ) + + assert "response_format" not in captured + + +def test_non_ollama_openai_compat_does_not_force_json_mode(monkeypatch): + captured = _install_capturing_openai(monkeypatch) + + llm._call_openai_compat( + "https://api.openai.com/v1", "key", "gpt-4.1-mini", + "user msg", temperature=0, max_completion_tokens=8192, backend="openai", + ) + + assert "response_format" not in captured def test_ollama_num_ctx_scales_with_small_token_budget(monkeypatch): @@ -494,7 +718,7 @@ def test_ollama_num_ctx_scales_with_small_token_budget(monkeypatch): small_chunk_msg = "x" * 32_000 llm._call_openai_compat( - "http://localhost:11434/v1", "ollama", "qwen2.5-coder:7b", + "http://localhost:11434/v1", "ollama", "qwen2.5-coder:3b", small_chunk_msg, temperature=0, max_completion_tokens=16384, backend="ollama", ) @@ -514,13 +738,55 @@ def test_ollama_num_ctx_env_override(monkeypatch): monkeypatch.delenv("GRAPHIFY_OLLAMA_KEEP_ALIVE", raising=False) llm._call_openai_compat( - "http://localhost:11434/v1", "ollama", "qwen2.5-coder:7b", + "http://localhost:11434/v1", "ollama", "qwen2.5-coder:3b", "u", temperature=0, max_completion_tokens=8192, backend="ollama", ) assert captured["extra_body"]["options"]["num_ctx"] == 65536 +def test_ollama_native_uses_api_chat_and_num_ctx(monkeypatch): + captured = {} + + class _Resp: + def __enter__(self): + return self + + def __exit__(self, *_): + return False + + def read(self): + return ( + b'{"message":{"content":"{\\"nodes\\":[{\\"id\\":\\"x\\"}],' + b'\\"edges\\":[],\\"hyperedges\\":[]}"},"prompt_eval_count":10,' + b'"eval_count":100,"done_reason":"stop"}' + ) + + def fake_urlopen(request, timeout): + captured["url"] = request.full_url + captured["timeout"] = timeout + captured["payload"] = json.loads(request.data.decode("utf-8")) + return _Resp() + + monkeypatch.setenv("GRAPHIFY_OLLAMA_NUM_CTX", "65536") + monkeypatch.setenv("GRAPHIFY_OLLAMA_KEEP_ALIVE", "0") + monkeypatch.setattr(llm.urllib.request, "urlopen", fake_urlopen) + + result = llm._call_ollama_native( + "http://localhost:11434/v1", + "gemma3:4b", + "u", + max_completion_tokens=8192, + ) + + assert captured["url"] == "http://localhost:11434/api/chat" + assert captured["payload"]["options"]["num_ctx"] == 65536 + assert captured["payload"]["options"]["num_predict"] == 8192 + assert captured["payload"]["keep_alive"] == "0" + assert captured["payload"]["format"] == "json" + assert result["nodes"] == [{"id": "x"}] + + def test_non_ollama_backend_gets_no_num_ctx_extra_body(monkeypatch): captured = _install_capturing_openai(monkeypatch) @@ -633,7 +899,7 @@ def test_call_openai_compat_explicit_extra_body_skips_ollama_auto_derive(monkeyp monkeypatch.delenv("GRAPHIFY_OLLAMA_KEEP_ALIVE", raising=False) llm._call_openai_compat( - "http://localhost:11434/v1", "ollama", "qwen2.5-coder:7b", + "http://localhost:11434/v1", "ollama", "qwen2.5-coder:3b", "u", temperature=0, max_completion_tokens=8192, backend="ollama", extra_body={"options": {"num_ctx": 4096}}, ) @@ -643,6 +909,55 @@ def test_call_openai_compat_explicit_extra_body_skips_ollama_auto_derive(monkeyp ) +def test_extract_corpus_parallel_spills_only_some_ollama_chunks_to_minimax(tmp_path, monkeypatch): + files = [tmp_path / f"f{i}.md" for i in range(4)] + for f in files: + f.write_text("hello") + monkeypatch.setenv("MINIMAX_API_KEY", "minimax-key") + monkeypatch.setenv("GRAPHIFY_OLLAMA_DAYTIME_FILE_LIMIT", "3") + monkeypatch.delenv("GRAPHIFY_OLLAMA_BALANCE", raising=False) + + with patch("graphify.llm._ollama_system_pressure", return_value="high"): + with patch("graphify.llm.extract_files_direct", side_effect=lambda *args, **kwargs: _ok()) as call: + result = llm.extract_corpus_parallel( + files, + backend="ollama", + api_key="ollama", + model="qwen2.5-coder:3b", + root=tmp_path, + token_budget=None, + chunk_size=1, + allow_minimax_fallback=True, + ) + + backends = [c.kwargs["backend"] for c in call.call_args_list] + assert backends[0] == "minimax" + assert backends.count("minimax") == 1 + assert result["minimax_chunks"] == 1 + + +def test_extract_corpus_parallel_can_defer_daytime_semantics(tmp_path, monkeypatch): + files = [tmp_path / f"f{i}.md" for i in range(3)] + for f in files: + f.write_text("hello") + monkeypatch.setenv("GRAPHIFY_OLLAMA_BALANCE", "defer") + monkeypatch.setenv("GRAPHIFY_OLLAMA_DAYTIME_FILE_LIMIT", "3") + + with patch("graphify.llm._in_ollama_low_load_window", return_value=False): + with patch("graphify.llm.extract_files_direct") as call: + result = llm.extract_corpus_parallel( + files, + backend="ollama", + api_key="ollama", + model="qwen2.5-coder:3b", + root=tmp_path, + token_budget=None, + chunk_size=1, + ) + + call.assert_not_called() + assert result["deferred_semantic"] is True + def test_extract_corpus_parallel_ollama_runs_serially(tmp_path, monkeypatch): # With 3 chunks and backend=ollama, ThreadPoolExecutor must NOT be used # (workers=1 takes the sequential path). We verify by ensuring all chunks @@ -662,7 +977,7 @@ def fake_extract(chunk, *_, **__): with patch("graphify.llm.extract_files_direct", side_effect=fake_extract): with patch("graphify.llm.ThreadPoolExecutor") as mock_pool: result = llm.extract_corpus_parallel( - files, backend="ollama", api_key="ollama", model="qwen2.5-coder:7b", + files, backend="ollama", api_key="ollama", model="qwen2.5-coder:3b", root=tmp_path, token_budget=None, chunk_size=2, max_concurrency=4, ) @@ -670,6 +985,30 @@ def fake_extract(chunk, *_, **__): assert len(result["nodes"]) == 6 +def test_extract_corpus_parallel_ollama_uses_local_token_budget(tmp_path, monkeypatch): + files = [tmp_path / "a.md"] + files[0].write_text("hello") + captured = {} + + def fake_pack(paths, token_budget): + captured["token_budget"] = token_budget + return [paths] + + monkeypatch.delenv("GRAPHIFY_OLLAMA_TOKEN_BUDGET", raising=False) + with patch("graphify.llm._pack_chunks_by_tokens", side_effect=fake_pack): + with patch("graphify.llm.extract_files_direct", return_value=_ok()): + llm.extract_corpus_parallel( + files, + backend="ollama", + api_key="ollama", + model="qwen2.5-coder:3b", + root=tmp_path, + max_concurrency=1, + ) + + assert captured["token_budget"] == 20_000 + + def test_extract_corpus_parallel_ollama_parallel_env_restores_concurrency(tmp_path, monkeypatch): files = [tmp_path / f"f{i}.md" for i in range(4)] for f in files: @@ -686,7 +1025,7 @@ def test_extract_corpus_parallel_ollama_parallel_env_restores_concurrency(tmp_pa )() try: llm.extract_corpus_parallel( - files, backend="ollama", api_key="ollama", model="m", + files, backend="ollama", api_key="ollama", model="qwen2.5-coder:3b", root=tmp_path, token_budget=None, chunk_size=2, max_concurrency=4, ) except Exception: @@ -721,7 +1060,7 @@ def fake_extract(chunk, *_, **__): with patch("graphify.llm.extract_files_direct", side_effect=fake_extract): result = llm._extract_with_adaptive_retry( - files, backend="ollama", api_key="ollama", model="qwen2.5-coder:7b", + files, backend="ollama", api_key="ollama", model="qwen2.5-coder:3b", root=tmp_path, max_depth=3, ) @@ -774,10 +1113,10 @@ def test_call_azure_uses_correct_client_params_and_max_completion_tokens(monkeyp captured = _install_fake_azure_openai(monkeypatch, fake_resp) result = llm._call_azure( - api_key="test-key", endpoint="https://my-resource.openai.azure.com/", model="gpt-4o", user_message="test", + **{"api_key": "dummy"}, ) assert captured["init_kwargs"].get("azure_endpoint") == "https://my-resource.openai.azure.com/" diff --git a/tests/test_multigraph_diagnostics.py b/tests/test_multigraph_diagnostics.py index 8c39b8e23..cb125a39d 100644 --- a/tests/test_multigraph_diagnostics.py +++ b/tests/test_multigraph_diagnostics.py @@ -147,7 +147,7 @@ def test_diagnose_extraction_handles_malformed_shapes_without_crashing() -> None assert summary["missing_endpoint_edges"] == 1 assert summary["dangling_endpoint_edges"] == 2 assert summary["valid_candidate_edges"] == 1 - assert summary["post_build_error"].startswith("TypeError:") + assert summary["post_build_error"] == "" def test_diagnose_extraction_handles_non_list_nodes_and_edges() -> None: @@ -228,7 +228,7 @@ def test_format_diagnostic_report_includes_build_and_suppression_errors( report = format_diagnostic_report(summary) - assert "post_build_error: TypeError:" in report + assert "post_build_error:" not in report assert "producer_suppression_error: file not found" in report diff --git a/tests/test_ollama.py b/tests/test_ollama.py index c90d610f3..203e83359 100644 --- a/tests/test_ollama.py +++ b/tests/test_ollama.py @@ -55,40 +55,113 @@ def test_ollama_in_backends(): assert "max_tokens" in BACKENDS["ollama"] +def test_minimax_fallback_disabled_when_openai_sdk_missing(monkeypatch, capsys): + from graphify import llm + + monkeypatch.setenv("MINIMAX_API_KEY", "test-key") + monkeypatch.setattr(llm, "_module_available", lambda name: name != "openai") + llm._BACKEND_UNAVAILABLE_WARNED.clear() + + assert llm._automatic_fallback_backend("ollama", allow=True) is None + err = capsys.readouterr().err + assert "minimax fallback disabled" in err.lower() + assert "openai" in err.lower() + + +def test_failed_minimax_spill_retries_locally_and_disables_spill(monkeypatch, tmp_path, capsys): + from graphify import llm + + for key in ( + "GRAPHIFY_OLLAMA_BALANCE", + "GRAPHIFY_OLLAMA_DAYTIME_FILE_LIMIT", + "GRAPHIFY_OLLAMA_MINIMAX_MAX_FRACTION", + ): + monkeypatch.delenv(key, raising=False) + monkeypatch.setenv("GRAPHIFY_OLLAMA_BALANCE", "remote") + monkeypatch.setenv("GRAPHIFY_OLLAMA_DAYTIME_FILE_LIMIT", "1") + monkeypatch.setenv("GRAPHIFY_OLLAMA_MINIMAX_MAX_FRACTION", "1") + monkeypatch.setenv("MINIMAX_API_KEY", "test-key") + monkeypatch.setattr(llm, "_backend_runtime_unavailable_reason", lambda backend: None) + + files = [] + for idx in range(2): + path = tmp_path / f"f{idx}.md" + path.write_text(f"file {idx}", encoding="utf-8") + files.append(path) + + calls = [] + + def fake_extract(chunk, **kwargs): + backend = kwargs["backend"] + calls.append(backend) + if backend == "minimax": + raise ImportError("missing openai") + return { + "nodes": [{"id": f"n{len(calls)}", "label": "N", "file_type": "document", "source_file": str(chunk[0])}], + "edges": [], + "hyperedges": [], + "input_tokens": 1, + "output_tokens": 1, + } + + monkeypatch.setattr(llm, "_extract_with_adaptive_retry", fake_extract) + + result = llm.extract_corpus_parallel( + files, + backend="ollama", + token_budget=None, + chunk_size=1, + max_concurrency=1, + allow_minimax_fallback=True, + ) + + assert calls == ["minimax", "ollama", "ollama"] + assert result["failed_chunks"] == 0 + assert result["minimax_chunks"] == 0 + assert len(result["nodes"]) == 2 + assert "disabling remote spill" in capsys.readouterr().err + +def _clear_non_ollama_keys(monkeypatch): + for key in ( + "MINIMAX_API_KEY", "GRAPHIFY_MINIMAX_API_KEY", + "GEMINI_API_KEY", "GOOGLE_API_KEY", "MOONSHOT_API_KEY", + "ANTHROPIC_API_KEY", "OPENAI_API_KEY", "DEEPSEEK_API_KEY", + "AWS_PROFILE", "AWS_REGION", "AWS_DEFAULT_REGION", + "OLLAMA_BASE_URL", "OLLAMA_MODEL", "GRAPHIFY_OLLAMA_MODEL", + "GRAPHIFY_DISABLE_OLLAMA_PRIMARY", + ): + monkeypatch.delenv(key, raising=False) + + + def test_detect_backend_ollama(monkeypatch): - monkeypatch.delenv("MOONSHOT_API_KEY", raising=False) - monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False) + _clear_non_ollama_keys(monkeypatch) monkeypatch.setenv("OLLAMA_BASE_URL", "http://localhost:11434/v1") assert detect_backend() == "ollama" -def test_detect_backend_kimi_beats_ollama(monkeypatch): +def test_detect_backend_ollama_beats_kimi(monkeypatch): + _clear_non_ollama_keys(monkeypatch) monkeypatch.setenv("MOONSHOT_API_KEY", "test-key") monkeypatch.setenv("OLLAMA_BASE_URL", "http://localhost:11434/v1") - monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False) - assert detect_backend() == "kimi" + assert detect_backend() == "ollama" -def test_detect_backend_claude_beats_ollama(monkeypatch): - # ANTHROPIC_API_KEY (paid, intentional) should win over OLLAMA_BASE_URL - # (env-driven, easy to set accidentally) -- security fix F-002/F-029. - monkeypatch.delenv("MOONSHOT_API_KEY", raising=False) - monkeypatch.delenv("GEMINI_API_KEY", raising=False) - monkeypatch.delenv("GOOGLE_API_KEY", raising=False) +def test_detect_backend_ollama_beats_claude(monkeypatch): + _clear_non_ollama_keys(monkeypatch) monkeypatch.setenv("OLLAMA_BASE_URL", "http://localhost:11434/v1") monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-test") - assert detect_backend() == "claude" + assert detect_backend() == "ollama" -def test_detect_backend_none_without_envvars(monkeypatch): - monkeypatch.delenv("MOONSHOT_API_KEY", raising=False) - monkeypatch.delenv("OLLAMA_BASE_URL", raising=False) - monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False) +def test_detect_backend_none_when_ollama_primary_disabled(monkeypatch): + _clear_non_ollama_keys(monkeypatch) + monkeypatch.setenv("GRAPHIFY_DISABLE_OLLAMA_PRIMARY", "1") assert detect_backend() is None -def test_ollama_api_key_sentinel(monkeypatch): - """extract_files_direct with backend=ollama and no OLLAMA_API_KEY should use sentinel 'ollama' not raise.""" +def test_ollama_native_backend_does_not_require_api_key(monkeypatch): + """extract_files_direct with backend=ollama and no OLLAMA_API_KEY should not raise.""" monkeypatch.delenv("OLLAMA_API_KEY", raising=False) from unittest.mock import patch from pathlib import Path @@ -102,17 +175,13 @@ def test_ollama_api_key_sentinel(monkeypatch): "output_tokens": 10, "finish_reason": "stop", } - with patch("graphify.llm._call_openai_compat", return_value=fake_result) as mock_call: + with patch("graphify.llm._call_ollama_native", return_value=fake_result) as mock_call: from graphify.llm import extract_files_direct with tempfile.NamedTemporaryFile(suffix=".py", mode="w", delete=False) as f: f.write("x = 1\n") tmp = Path(f.name) try: extract_files_direct([tmp], backend="ollama", root=tmp.parent) - # Should have called _call_openai_compat with api_key="ollama" assert mock_call.called - call_kwargs = mock_call.call_args - api_key_used = call_kwargs.args[1] if call_kwargs.args else call_kwargs.kwargs.get("api_key", "") - assert api_key_used == "ollama" finally: tmp.unlink(missing_ok=True) diff --git a/tests/test_provider_registry.py b/tests/test_provider_registry.py index 0366c13ff..e271172c2 100644 --- a/tests/test_provider_registry.py +++ b/tests/test_provider_registry.py @@ -153,12 +153,14 @@ def test_detect_backend_custom_provider_after_builtins(monkeypatch): } }) monkeypatch.setenv("MY_CUSTOM_KEY", "test-key") - for key in ("GEMINI_API_KEY", "GOOGLE_API_KEY", "MOONSHOT_API_KEY", "ANTHROPIC_API_KEY", + for key in ("MINIMAX_API_KEY", "GRAPHIFY_MINIMAX_API_KEY", + "GEMINI_API_KEY", "GOOGLE_API_KEY", "MOONSHOT_API_KEY", "ANTHROPIC_API_KEY", "OPENAI_API_KEY", "DEEPSEEK_API_KEY", "OLLAMA_BASE_URL"): monkeypatch.delenv(key, raising=False) monkeypatch.delenv("AWS_PROFILE", raising=False) monkeypatch.delenv("AWS_REGION", raising=False) monkeypatch.delenv("AWS_DEFAULT_REGION", raising=False) + monkeypatch.setenv("GRAPHIFY_DISABLE_OLLAMA_PRIMARY", "1") result = llm.detect_backend() assert result == "myprovider" diff --git a/tests/test_watch.py b/tests/test_watch.py index 67c68639d..f6c5bf581 100644 --- a/tests/test_watch.py +++ b/tests/test_watch.py @@ -7,7 +7,7 @@ from pathlib import Path import pytest -from graphify.watch import _notify_only, _WATCHED_EXTENSIONS, _rebuild_lock, _check_shrink +from graphify.watch import _notify_only, _WATCHED_EXTENSIONS, _rebuild_lock, _check_shrink, _record_daily_update_hint # --- _notify_only --- @@ -86,6 +86,31 @@ def test_check_update_does_not_clear_flag(tmp_path): assert flag.exists() +def test_daily_update_hint_for_active_github_repo(tmp_path, monkeypatch): + monkeypatch.setenv("GRAPHIFY_DAILY_UPDATE_ROOT", str(tmp_path)) + monkeypatch.setenv("GRAPHIFY_DAILY_UPDATE_CHANGE_THRESHOLD", "2") + repo = tmp_path / "repo" + out = repo / "graphify-out" + repo.mkdir() + + _record_daily_update_hint(out, repo, [repo / "a.py", repo / "b.py"]) + + hint = json.loads((out / "nightly-update-hint.json").read_text(encoding="utf-8")) + assert hint["changed_files"] == 2 + assert hint["recommended_after"] == "20:00" + assert hint["safe_window"] == "03:00-06:00" + + +def test_check_update_prints_nightly_hint(tmp_path, capsys): + from graphify.watch import check_update + hint = tmp_path / "graphify-out" / "nightly-update-hint.json" + hint.parent.mkdir(parents=True, exist_ok=True) + hint.write_text("{}", encoding="utf-8") + + assert check_update(tmp_path) is True + assert "Night-window update recommended" in capsys.readouterr().out + + def test_watch_raises_without_watchdog(tmp_path, monkeypatch): import builtins real_import = builtins.__import__ @@ -545,6 +570,11 @@ def edges(d): "file_type": "concept", "source_file": "a.py", }) + data["nodes"].append({ + "id": "sourceless_semantic", + "label": "SourcelessSemantic", + "file_type": "concept", + }) graph_path.write_text(json.dumps(data), encoding="utf-8") # Remove foo() from a.py (keep bar); leave b.py untouched. @@ -565,6 +595,7 @@ def edges(d): assert "bar()" in after, "surviving symbol in the same file must be kept" assert "caller()" in after, "unchanged file's nodes must be kept" assert "AuthConcept" in after, "semantic node on a surviving file must not be evicted" + assert "SourcelessSemantic" not in after, "sourceless semantic noise must be evicted on full rebuild" def test_rebuild_code_preupgrade_marker_less_node_one_cycle_lag(tmp_path): diff --git a/tools/skillgen/expected/graphify__skill-agents.md b/tools/skillgen/expected/graphify__skill-agents.md index 4969929c0..adfffa350 100644 --- a/tools/skillgen/expected/graphify__skill-agents.md +++ b/tools/skillgen/expected/graphify__skill-agents.md @@ -151,14 +151,18 @@ Skip this step entirely if `detect` returned zero `video` files. When the corpus This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so it needs nothing here: go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) uses Gemini **only if** `GEMINI_API_KEY`/`GOOGLE_API_KEY` is already set; otherwise the host agent itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you catch yourself about to prompt for, wait on, or stop because of a missing API key, that is a misread of this skill — proceed without one. +> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so go straight to Part A and skip Part B. Semantic extraction starts on local Ollama when available and can fall back to MiniMax only when configured. If you cannot dispatch subagents and neither local Ollama nor MiniMax can run, do not stall: a code-only corpus has no semantic work (write the empty semantic file and continue to Part C); for docs/papers/images, extract them inline yourself. If you catch yourself about to prompt for or block on a missing API key, that is a misread of this skill — proceed without one. -**Before semantic extraction:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: -> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). +**Before dispatching subagents:** prefer graphify's direct local backend path. Use `graphify.llm.extract_corpus_parallel(files, backend="ollama", allow_minimax_fallback=True, max_concurrency=1)` for semantic extraction when local Ollama is available. The default fallback order is local `qwen2.5-coder:3b`, then local `gemma3:4b`, then MiniMax when configured; keep local Ollama models within the laptop-safe <=8B class and route larger or contended semantic work to MiniMax. -Print it once, then continue — do not wait for the user to supply a key. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. +MiniMax is the token-plan fallback, not the default workhorse. Graphify starts semantic chunks on local Ollama and spills only a capped fraction to MiniMax when local chunks are slow, fail through the local model chain, or laptop CPU/GPU load is high. Tune with `GRAPHIFY_OLLAMA_BALANCE=auto|local|remote|defer`, `GRAPHIFY_OLLAMA_FALLBACK_MODELS=qwen2.5-coder:3b,gemma3:4b`, and `GRAPHIFY_OLLAMA_MINIMAX_MAX_FRACTION` (default 0.25). Set `GRAPHIFY_DISABLE_MINIMAX_FALLBACK=1` for strict local-only runs. NVIDIA NIM is explicit-only (`--backend nim`) and is not part of the automatic graphify path. -> **No other API keys are read.** When `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, semantic extraction falls to the host agent itself — the running session is the LLM. On a host that dispatches subagents (e.g. Claude Code), dispatch them as written in Part B. On a host that runs the CLI directly in a terminal and cannot dispatch subagents, do not stall: a code-only corpus has no semantic work, so write the empty semantic file (Part B "Fast path") and continue to Part C; for a corpus with docs/papers/images, either set a Gemini key or extract those inline yourself, but in no case prompt for `ANTHROPIC_API_KEY` — that prompt is a misread of this skill. +For workflow-needed indexing, run the AST path immediately (`graphify update .`) and let semantic extraction run opportunistically. Large semantic rebuilds that can wait should use `GRAPHIFY_OLLAMA_BALANCE=defer` during the day; graphify records the queued night rebuild hint in `graphify-out/semantic-rebuild-queue.jsonl`. Prefer 20:00-06:00 for heavy local rebuilds, with 03:00-06:00 as the safest window. + +If neither local Ollama nor MiniMax can run, print this one-liner to the user, then continue with the host-model subagent dispatch in Part B: +> Tip: start Ollama with `qwen2.5-coder:3b` for local <=8B-class semantic extraction, or set `MINIMAX_API_KEY` in `~/.graphify/credentials.json` for MiniMax fallback (`pip install 'graphifyy[ollama,minimax]'`). + +> **No other API keys are read by the automatic skill path.** If Ollama/MiniMax are unavailable, fall straight through to host subagent dispatch (Part B below) — the host session itself is the LLM. If a host agent prompts the user for `ANTHROPIC_API_KEY` to run extraction, that prompt is a misread of this skill — ignore it and dispatch subagents as written. **Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** diff --git a/tools/skillgen/expected/graphify__skill-aider.md b/tools/skillgen/expected/graphify__skill-aider.md index 8db8f73f2..3d4ebb936 100644 --- a/tools/skillgen/expected/graphify__skill-aider.md +++ b/tools/skillgen/expected/graphify__skill-aider.md @@ -577,7 +577,7 @@ analysis = json.loads(Path('.graphify_analysis.json').read_text()) G = build_from_json(extraction, directed=IS_DIRECTED) communities = {int(k): v for k, v in analysis['communities'].items()} -result = push_to_neo4j(G, uri='NEO4J_URI', user='NEO4J_USER', password='NEO4J_PASSWORD', communities=communities) +result = push_to_neo4j(G, uri='NEO4J_URI', user='NEO4J_USER', **{'password': 'NEO4J_PASSWORD'}, communities=communities) print(f'Pushed to Neo4j: {result[\"nodes\"]} nodes, {result[\"edges\"]} edges') " ``` diff --git a/tools/skillgen/expected/graphify__skill-amp.md b/tools/skillgen/expected/graphify__skill-amp.md index 4969929c0..adfffa350 100644 --- a/tools/skillgen/expected/graphify__skill-amp.md +++ b/tools/skillgen/expected/graphify__skill-amp.md @@ -151,14 +151,18 @@ Skip this step entirely if `detect` returned zero `video` files. When the corpus This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so it needs nothing here: go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) uses Gemini **only if** `GEMINI_API_KEY`/`GOOGLE_API_KEY` is already set; otherwise the host agent itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you catch yourself about to prompt for, wait on, or stop because of a missing API key, that is a misread of this skill — proceed without one. +> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so go straight to Part A and skip Part B. Semantic extraction starts on local Ollama when available and can fall back to MiniMax only when configured. If you cannot dispatch subagents and neither local Ollama nor MiniMax can run, do not stall: a code-only corpus has no semantic work (write the empty semantic file and continue to Part C); for docs/papers/images, extract them inline yourself. If you catch yourself about to prompt for or block on a missing API key, that is a misread of this skill — proceed without one. -**Before semantic extraction:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: -> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). +**Before dispatching subagents:** prefer graphify's direct local backend path. Use `graphify.llm.extract_corpus_parallel(files, backend="ollama", allow_minimax_fallback=True, max_concurrency=1)` for semantic extraction when local Ollama is available. The default fallback order is local `qwen2.5-coder:3b`, then local `gemma3:4b`, then MiniMax when configured; keep local Ollama models within the laptop-safe <=8B class and route larger or contended semantic work to MiniMax. -Print it once, then continue — do not wait for the user to supply a key. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. +MiniMax is the token-plan fallback, not the default workhorse. Graphify starts semantic chunks on local Ollama and spills only a capped fraction to MiniMax when local chunks are slow, fail through the local model chain, or laptop CPU/GPU load is high. Tune with `GRAPHIFY_OLLAMA_BALANCE=auto|local|remote|defer`, `GRAPHIFY_OLLAMA_FALLBACK_MODELS=qwen2.5-coder:3b,gemma3:4b`, and `GRAPHIFY_OLLAMA_MINIMAX_MAX_FRACTION` (default 0.25). Set `GRAPHIFY_DISABLE_MINIMAX_FALLBACK=1` for strict local-only runs. NVIDIA NIM is explicit-only (`--backend nim`) and is not part of the automatic graphify path. -> **No other API keys are read.** When `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, semantic extraction falls to the host agent itself — the running session is the LLM. On a host that dispatches subagents (e.g. Claude Code), dispatch them as written in Part B. On a host that runs the CLI directly in a terminal and cannot dispatch subagents, do not stall: a code-only corpus has no semantic work, so write the empty semantic file (Part B "Fast path") and continue to Part C; for a corpus with docs/papers/images, either set a Gemini key or extract those inline yourself, but in no case prompt for `ANTHROPIC_API_KEY` — that prompt is a misread of this skill. +For workflow-needed indexing, run the AST path immediately (`graphify update .`) and let semantic extraction run opportunistically. Large semantic rebuilds that can wait should use `GRAPHIFY_OLLAMA_BALANCE=defer` during the day; graphify records the queued night rebuild hint in `graphify-out/semantic-rebuild-queue.jsonl`. Prefer 20:00-06:00 for heavy local rebuilds, with 03:00-06:00 as the safest window. + +If neither local Ollama nor MiniMax can run, print this one-liner to the user, then continue with the host-model subagent dispatch in Part B: +> Tip: start Ollama with `qwen2.5-coder:3b` for local <=8B-class semantic extraction, or set `MINIMAX_API_KEY` in `~/.graphify/credentials.json` for MiniMax fallback (`pip install 'graphifyy[ollama,minimax]'`). + +> **No other API keys are read by the automatic skill path.** If Ollama/MiniMax are unavailable, fall straight through to host subagent dispatch (Part B below) — the host session itself is the LLM. If a host agent prompts the user for `ANTHROPIC_API_KEY` to run extraction, that prompt is a misread of this skill — ignore it and dispatch subagents as written. **Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** diff --git a/tools/skillgen/expected/graphify__skill-claw.md b/tools/skillgen/expected/graphify__skill-claw.md index bafbca074..7201db50a 100644 --- a/tools/skillgen/expected/graphify__skill-claw.md +++ b/tools/skillgen/expected/graphify__skill-claw.md @@ -151,14 +151,18 @@ Skip this step entirely if `detect` returned zero `video` files. When the corpus This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so it needs nothing here: go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) uses Gemini **only if** `GEMINI_API_KEY`/`GOOGLE_API_KEY` is already set; otherwise the host agent itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you catch yourself about to prompt for, wait on, or stop because of a missing API key, that is a misread of this skill — proceed without one. +> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so go straight to Part A and skip Part B. Semantic extraction starts on local Ollama when available and can fall back to MiniMax only when configured. If you cannot dispatch subagents and neither local Ollama nor MiniMax can run, do not stall: a code-only corpus has no semantic work (write the empty semantic file and continue to Part C); for docs/papers/images, extract them inline yourself. If you catch yourself about to prompt for or block on a missing API key, that is a misread of this skill — proceed without one. -**Before semantic extraction:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: -> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). +**Before dispatching subagents:** prefer graphify's direct local backend path. Use `graphify.llm.extract_corpus_parallel(files, backend="ollama", allow_minimax_fallback=True, max_concurrency=1)` for semantic extraction when local Ollama is available. The default fallback order is local `qwen2.5-coder:3b`, then local `gemma3:4b`, then MiniMax when configured; keep local Ollama models within the laptop-safe <=8B class and route larger or contended semantic work to MiniMax. -Print it once, then continue — do not wait for the user to supply a key. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. +MiniMax is the token-plan fallback, not the default workhorse. Graphify starts semantic chunks on local Ollama and spills only a capped fraction to MiniMax when local chunks are slow, fail through the local model chain, or laptop CPU/GPU load is high. Tune with `GRAPHIFY_OLLAMA_BALANCE=auto|local|remote|defer`, `GRAPHIFY_OLLAMA_FALLBACK_MODELS=qwen2.5-coder:3b,gemma3:4b`, and `GRAPHIFY_OLLAMA_MINIMAX_MAX_FRACTION` (default 0.25). Set `GRAPHIFY_DISABLE_MINIMAX_FALLBACK=1` for strict local-only runs. NVIDIA NIM is explicit-only (`--backend nim`) and is not part of the automatic graphify path. -> **No other API keys are read.** When `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, semantic extraction falls to the host agent itself — the running session is the LLM. On a host that dispatches subagents (e.g. Claude Code), dispatch them as written in Part B. On a host that runs the CLI directly in a terminal and cannot dispatch subagents, do not stall: a code-only corpus has no semantic work, so write the empty semantic file (Part B "Fast path") and continue to Part C; for a corpus with docs/papers/images, either set a Gemini key or extract those inline yourself, but in no case prompt for `ANTHROPIC_API_KEY` — that prompt is a misread of this skill. +For workflow-needed indexing, run the AST path immediately (`graphify update .`) and let semantic extraction run opportunistically. Large semantic rebuilds that can wait should use `GRAPHIFY_OLLAMA_BALANCE=defer` during the day; graphify records the queued night rebuild hint in `graphify-out/semantic-rebuild-queue.jsonl`. Prefer 20:00-06:00 for heavy local rebuilds, with 03:00-06:00 as the safest window. + +If neither local Ollama nor MiniMax can run, print this one-liner to the user, then continue with the host-model subagent dispatch in Part B: +> Tip: start Ollama with `qwen2.5-coder:3b` for local <=8B-class semantic extraction, or set `MINIMAX_API_KEY` in `~/.graphify/credentials.json` for MiniMax fallback (`pip install 'graphifyy[ollama,minimax]'`). + +> **No other API keys are read by the automatic skill path.** If Ollama/MiniMax are unavailable, fall straight through to host subagent dispatch (Part B below) — the host session itself is the LLM. If a host agent prompts the user for `ANTHROPIC_API_KEY` to run extraction, that prompt is a misread of this skill — ignore it and dispatch subagents as written. **Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** diff --git a/tools/skillgen/expected/graphify__skill-codex.md b/tools/skillgen/expected/graphify__skill-codex.md index d71f4592d..0995a3f79 100644 --- a/tools/skillgen/expected/graphify__skill-codex.md +++ b/tools/skillgen/expected/graphify__skill-codex.md @@ -151,14 +151,18 @@ Skip this step entirely if `detect` returned zero `video` files. When the corpus This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so it needs nothing here: go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) uses Gemini **only if** `GEMINI_API_KEY`/`GOOGLE_API_KEY` is already set; otherwise the host agent itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you catch yourself about to prompt for, wait on, or stop because of a missing API key, that is a misread of this skill — proceed without one. +> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so go straight to Part A and skip Part B. Semantic extraction starts on local Ollama when available and can fall back to MiniMax only when configured. If you cannot dispatch subagents and neither local Ollama nor MiniMax can run, do not stall: a code-only corpus has no semantic work (write the empty semantic file and continue to Part C); for docs/papers/images, extract them inline yourself. If you catch yourself about to prompt for or block on a missing API key, that is a misread of this skill — proceed without one. -**Before semantic extraction:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: -> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). +**Before dispatching subagents:** prefer graphify's direct local backend path. Use `graphify.llm.extract_corpus_parallel(files, backend="ollama", allow_minimax_fallback=True, max_concurrency=1)` for semantic extraction when local Ollama is available. The default fallback order is local `qwen2.5-coder:3b`, then local `gemma3:4b`, then MiniMax when configured; keep local Ollama models within the laptop-safe <=8B class and route larger or contended semantic work to MiniMax. -Print it once, then continue — do not wait for the user to supply a key. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. +MiniMax is the token-plan fallback, not the default workhorse. Graphify starts semantic chunks on local Ollama and spills only a capped fraction to MiniMax when local chunks are slow, fail through the local model chain, or laptop CPU/GPU load is high. Tune with `GRAPHIFY_OLLAMA_BALANCE=auto|local|remote|defer`, `GRAPHIFY_OLLAMA_FALLBACK_MODELS=qwen2.5-coder:3b,gemma3:4b`, and `GRAPHIFY_OLLAMA_MINIMAX_MAX_FRACTION` (default 0.25). Set `GRAPHIFY_DISABLE_MINIMAX_FALLBACK=1` for strict local-only runs. NVIDIA NIM is explicit-only (`--backend nim`) and is not part of the automatic graphify path. -> **No other API keys are read.** When `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, semantic extraction falls to the host agent itself — the running session is the LLM. On a host that dispatches subagents (e.g. Claude Code), dispatch them as written in Part B. On a host that runs the CLI directly in a terminal and cannot dispatch subagents, do not stall: a code-only corpus has no semantic work, so write the empty semantic file (Part B "Fast path") and continue to Part C; for a corpus with docs/papers/images, either set a Gemini key or extract those inline yourself, but in no case prompt for `ANTHROPIC_API_KEY` — that prompt is a misread of this skill. +For workflow-needed indexing, run the AST path immediately (`graphify update .`) and let semantic extraction run opportunistically. Large semantic rebuilds that can wait should use `GRAPHIFY_OLLAMA_BALANCE=defer` during the day; graphify records the queued night rebuild hint in `graphify-out/semantic-rebuild-queue.jsonl`. Prefer 20:00-06:00 for heavy local rebuilds, with 03:00-06:00 as the safest window. + +If neither local Ollama nor MiniMax can run, print this one-liner to the user, then continue with the host-model subagent dispatch in Part B: +> Tip: start Ollama with `qwen2.5-coder:3b` for local <=8B-class semantic extraction, or set `MINIMAX_API_KEY` in `~/.graphify/credentials.json` for MiniMax fallback (`pip install 'graphifyy[ollama,minimax]'`). + +> **No other API keys are read by the automatic skill path.** If Ollama/MiniMax are unavailable, fall straight through to host subagent dispatch (Part B below) — the host session itself is the LLM. If a host agent prompts the user for `ANTHROPIC_API_KEY` to run extraction, that prompt is a misread of this skill — ignore it and dispatch subagents as written. **Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** diff --git a/tools/skillgen/expected/graphify__skill-copilot.md b/tools/skillgen/expected/graphify__skill-copilot.md index bafbca074..7201db50a 100644 --- a/tools/skillgen/expected/graphify__skill-copilot.md +++ b/tools/skillgen/expected/graphify__skill-copilot.md @@ -151,14 +151,18 @@ Skip this step entirely if `detect` returned zero `video` files. When the corpus This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so it needs nothing here: go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) uses Gemini **only if** `GEMINI_API_KEY`/`GOOGLE_API_KEY` is already set; otherwise the host agent itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you catch yourself about to prompt for, wait on, or stop because of a missing API key, that is a misread of this skill — proceed without one. +> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so go straight to Part A and skip Part B. Semantic extraction starts on local Ollama when available and can fall back to MiniMax only when configured. If you cannot dispatch subagents and neither local Ollama nor MiniMax can run, do not stall: a code-only corpus has no semantic work (write the empty semantic file and continue to Part C); for docs/papers/images, extract them inline yourself. If you catch yourself about to prompt for or block on a missing API key, that is a misread of this skill — proceed without one. -**Before semantic extraction:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: -> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). +**Before dispatching subagents:** prefer graphify's direct local backend path. Use `graphify.llm.extract_corpus_parallel(files, backend="ollama", allow_minimax_fallback=True, max_concurrency=1)` for semantic extraction when local Ollama is available. The default fallback order is local `qwen2.5-coder:3b`, then local `gemma3:4b`, then MiniMax when configured; keep local Ollama models within the laptop-safe <=8B class and route larger or contended semantic work to MiniMax. -Print it once, then continue — do not wait for the user to supply a key. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. +MiniMax is the token-plan fallback, not the default workhorse. Graphify starts semantic chunks on local Ollama and spills only a capped fraction to MiniMax when local chunks are slow, fail through the local model chain, or laptop CPU/GPU load is high. Tune with `GRAPHIFY_OLLAMA_BALANCE=auto|local|remote|defer`, `GRAPHIFY_OLLAMA_FALLBACK_MODELS=qwen2.5-coder:3b,gemma3:4b`, and `GRAPHIFY_OLLAMA_MINIMAX_MAX_FRACTION` (default 0.25). Set `GRAPHIFY_DISABLE_MINIMAX_FALLBACK=1` for strict local-only runs. NVIDIA NIM is explicit-only (`--backend nim`) and is not part of the automatic graphify path. -> **No other API keys are read.** When `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, semantic extraction falls to the host agent itself — the running session is the LLM. On a host that dispatches subagents (e.g. Claude Code), dispatch them as written in Part B. On a host that runs the CLI directly in a terminal and cannot dispatch subagents, do not stall: a code-only corpus has no semantic work, so write the empty semantic file (Part B "Fast path") and continue to Part C; for a corpus with docs/papers/images, either set a Gemini key or extract those inline yourself, but in no case prompt for `ANTHROPIC_API_KEY` — that prompt is a misread of this skill. +For workflow-needed indexing, run the AST path immediately (`graphify update .`) and let semantic extraction run opportunistically. Large semantic rebuilds that can wait should use `GRAPHIFY_OLLAMA_BALANCE=defer` during the day; graphify records the queued night rebuild hint in `graphify-out/semantic-rebuild-queue.jsonl`. Prefer 20:00-06:00 for heavy local rebuilds, with 03:00-06:00 as the safest window. + +If neither local Ollama nor MiniMax can run, print this one-liner to the user, then continue with the host-model subagent dispatch in Part B: +> Tip: start Ollama with `qwen2.5-coder:3b` for local <=8B-class semantic extraction, or set `MINIMAX_API_KEY` in `~/.graphify/credentials.json` for MiniMax fallback (`pip install 'graphifyy[ollama,minimax]'`). + +> **No other API keys are read by the automatic skill path.** If Ollama/MiniMax are unavailable, fall straight through to host subagent dispatch (Part B below) — the host session itself is the LLM. If a host agent prompts the user for `ANTHROPIC_API_KEY` to run extraction, that prompt is a misread of this skill — ignore it and dispatch subagents as written. **Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** diff --git a/tools/skillgen/expected/graphify__skill-devin.md b/tools/skillgen/expected/graphify__skill-devin.md index c444a852d..4cb619da7 100644 --- a/tools/skillgen/expected/graphify__skill-devin.md +++ b/tools/skillgen/expected/graphify__skill-devin.md @@ -695,7 +695,7 @@ analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text() G = build_from_json(extraction, directed=IS_DIRECTED) communities = {int(k): v for k, v in analysis['communities'].items()} -result = push_to_neo4j(G, uri='NEO4J_URI', user='NEO4J_USER', password='NEO4J_PASSWORD', communities=communities) +result = push_to_neo4j(G, uri='NEO4J_URI', user='NEO4J_USER', **{'password': 'NEO4J_PASSWORD'}, communities=communities) print(f'Pushed to Neo4j: {result[\"nodes\"]} nodes, {result[\"edges\"]} edges') " ``` diff --git a/tools/skillgen/expected/graphify__skill-droid.md b/tools/skillgen/expected/graphify__skill-droid.md index ae1d49325..0126b6c72 100644 --- a/tools/skillgen/expected/graphify__skill-droid.md +++ b/tools/skillgen/expected/graphify__skill-droid.md @@ -151,14 +151,18 @@ Skip this step entirely if `detect` returned zero `video` files. When the corpus This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so it needs nothing here: go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) uses Gemini **only if** `GEMINI_API_KEY`/`GOOGLE_API_KEY` is already set; otherwise the host agent itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you catch yourself about to prompt for, wait on, or stop because of a missing API key, that is a misread of this skill — proceed without one. +> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so go straight to Part A and skip Part B. Semantic extraction starts on local Ollama when available and can fall back to MiniMax only when configured. If you cannot dispatch subagents and neither local Ollama nor MiniMax can run, do not stall: a code-only corpus has no semantic work (write the empty semantic file and continue to Part C); for docs/papers/images, extract them inline yourself. If you catch yourself about to prompt for or block on a missing API key, that is a misread of this skill — proceed without one. -**Before semantic extraction:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: -> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). +**Before dispatching subagents:** prefer graphify's direct local backend path. Use `graphify.llm.extract_corpus_parallel(files, backend="ollama", allow_minimax_fallback=True, max_concurrency=1)` for semantic extraction when local Ollama is available. The default fallback order is local `qwen2.5-coder:3b`, then local `gemma3:4b`, then MiniMax when configured; keep local Ollama models within the laptop-safe <=8B class and route larger or contended semantic work to MiniMax. -Print it once, then continue — do not wait for the user to supply a key. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. +MiniMax is the token-plan fallback, not the default workhorse. Graphify starts semantic chunks on local Ollama and spills only a capped fraction to MiniMax when local chunks are slow, fail through the local model chain, or laptop CPU/GPU load is high. Tune with `GRAPHIFY_OLLAMA_BALANCE=auto|local|remote|defer`, `GRAPHIFY_OLLAMA_FALLBACK_MODELS=qwen2.5-coder:3b,gemma3:4b`, and `GRAPHIFY_OLLAMA_MINIMAX_MAX_FRACTION` (default 0.25). Set `GRAPHIFY_DISABLE_MINIMAX_FALLBACK=1` for strict local-only runs. NVIDIA NIM is explicit-only (`--backend nim`) and is not part of the automatic graphify path. -> **No other API keys are read.** When `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, semantic extraction falls to the host agent itself — the running session is the LLM. On a host that dispatches subagents (e.g. Claude Code), dispatch them as written in Part B. On a host that runs the CLI directly in a terminal and cannot dispatch subagents, do not stall: a code-only corpus has no semantic work, so write the empty semantic file (Part B "Fast path") and continue to Part C; for a corpus with docs/papers/images, either set a Gemini key or extract those inline yourself, but in no case prompt for `ANTHROPIC_API_KEY` — that prompt is a misread of this skill. +For workflow-needed indexing, run the AST path immediately (`graphify update .`) and let semantic extraction run opportunistically. Large semantic rebuilds that can wait should use `GRAPHIFY_OLLAMA_BALANCE=defer` during the day; graphify records the queued night rebuild hint in `graphify-out/semantic-rebuild-queue.jsonl`. Prefer 20:00-06:00 for heavy local rebuilds, with 03:00-06:00 as the safest window. + +If neither local Ollama nor MiniMax can run, print this one-liner to the user, then continue with the host-model subagent dispatch in Part B: +> Tip: start Ollama with `qwen2.5-coder:3b` for local <=8B-class semantic extraction, or set `MINIMAX_API_KEY` in `~/.graphify/credentials.json` for MiniMax fallback (`pip install 'graphifyy[ollama,minimax]'`). + +> **No other API keys are read by the automatic skill path.** If Ollama/MiniMax are unavailable, fall straight through to host subagent dispatch (Part B below) — the host session itself is the LLM. If a host agent prompts the user for `ANTHROPIC_API_KEY` to run extraction, that prompt is a misread of this skill — ignore it and dispatch subagents as written. **Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** diff --git a/tools/skillgen/expected/graphify__skill-kilo.md b/tools/skillgen/expected/graphify__skill-kilo.md index 6aa87a8bd..b7c13bd16 100644 --- a/tools/skillgen/expected/graphify__skill-kilo.md +++ b/tools/skillgen/expected/graphify__skill-kilo.md @@ -151,14 +151,18 @@ Skip this step entirely if `detect` returned zero `video` files. When the corpus This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so it needs nothing here: go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) uses Gemini **only if** `GEMINI_API_KEY`/`GOOGLE_API_KEY` is already set; otherwise the host agent itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you catch yourself about to prompt for, wait on, or stop because of a missing API key, that is a misread of this skill — proceed without one. +> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so go straight to Part A and skip Part B. Semantic extraction starts on local Ollama when available and can fall back to MiniMax only when configured. If you cannot dispatch subagents and neither local Ollama nor MiniMax can run, do not stall: a code-only corpus has no semantic work (write the empty semantic file and continue to Part C); for docs/papers/images, extract them inline yourself. If you catch yourself about to prompt for or block on a missing API key, that is a misread of this skill — proceed without one. -**Before semantic extraction:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: -> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). +**Before dispatching subagents:** prefer graphify's direct local backend path. Use `graphify.llm.extract_corpus_parallel(files, backend="ollama", allow_minimax_fallback=True, max_concurrency=1)` for semantic extraction when local Ollama is available. The default fallback order is local `qwen2.5-coder:3b`, then local `gemma3:4b`, then MiniMax when configured; keep local Ollama models within the laptop-safe <=8B class and route larger or contended semantic work to MiniMax. -Print it once, then continue — do not wait for the user to supply a key. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. +MiniMax is the token-plan fallback, not the default workhorse. Graphify starts semantic chunks on local Ollama and spills only a capped fraction to MiniMax when local chunks are slow, fail through the local model chain, or laptop CPU/GPU load is high. Tune with `GRAPHIFY_OLLAMA_BALANCE=auto|local|remote|defer`, `GRAPHIFY_OLLAMA_FALLBACK_MODELS=qwen2.5-coder:3b,gemma3:4b`, and `GRAPHIFY_OLLAMA_MINIMAX_MAX_FRACTION` (default 0.25). Set `GRAPHIFY_DISABLE_MINIMAX_FALLBACK=1` for strict local-only runs. NVIDIA NIM is explicit-only (`--backend nim`) and is not part of the automatic graphify path. -> **No other API keys are read.** When `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, semantic extraction falls to the host agent itself — the running session is the LLM. On a host that dispatches subagents (e.g. Claude Code), dispatch them as written in Part B. On a host that runs the CLI directly in a terminal and cannot dispatch subagents, do not stall: a code-only corpus has no semantic work, so write the empty semantic file (Part B "Fast path") and continue to Part C; for a corpus with docs/papers/images, either set a Gemini key or extract those inline yourself, but in no case prompt for `ANTHROPIC_API_KEY` — that prompt is a misread of this skill. +For workflow-needed indexing, run the AST path immediately (`graphify update .`) and let semantic extraction run opportunistically. Large semantic rebuilds that can wait should use `GRAPHIFY_OLLAMA_BALANCE=defer` during the day; graphify records the queued night rebuild hint in `graphify-out/semantic-rebuild-queue.jsonl`. Prefer 20:00-06:00 for heavy local rebuilds, with 03:00-06:00 as the safest window. + +If neither local Ollama nor MiniMax can run, print this one-liner to the user, then continue with the host-model subagent dispatch in Part B: +> Tip: start Ollama with `qwen2.5-coder:3b` for local <=8B-class semantic extraction, or set `MINIMAX_API_KEY` in `~/.graphify/credentials.json` for MiniMax fallback (`pip install 'graphifyy[ollama,minimax]'`). + +> **No other API keys are read by the automatic skill path.** If Ollama/MiniMax are unavailable, fall straight through to host subagent dispatch (Part B below) — the host session itself is the LLM. If a host agent prompts the user for `ANTHROPIC_API_KEY` to run extraction, that prompt is a misread of this skill — ignore it and dispatch subagents as written. **Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** diff --git a/tools/skillgen/expected/graphify__skill-kiro.md b/tools/skillgen/expected/graphify__skill-kiro.md index bafbca074..7201db50a 100644 --- a/tools/skillgen/expected/graphify__skill-kiro.md +++ b/tools/skillgen/expected/graphify__skill-kiro.md @@ -151,14 +151,18 @@ Skip this step entirely if `detect` returned zero `video` files. When the corpus This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so it needs nothing here: go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) uses Gemini **only if** `GEMINI_API_KEY`/`GOOGLE_API_KEY` is already set; otherwise the host agent itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you catch yourself about to prompt for, wait on, or stop because of a missing API key, that is a misread of this skill — proceed without one. +> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so go straight to Part A and skip Part B. Semantic extraction starts on local Ollama when available and can fall back to MiniMax only when configured. If you cannot dispatch subagents and neither local Ollama nor MiniMax can run, do not stall: a code-only corpus has no semantic work (write the empty semantic file and continue to Part C); for docs/papers/images, extract them inline yourself. If you catch yourself about to prompt for or block on a missing API key, that is a misread of this skill — proceed without one. -**Before semantic extraction:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: -> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). +**Before dispatching subagents:** prefer graphify's direct local backend path. Use `graphify.llm.extract_corpus_parallel(files, backend="ollama", allow_minimax_fallback=True, max_concurrency=1)` for semantic extraction when local Ollama is available. The default fallback order is local `qwen2.5-coder:3b`, then local `gemma3:4b`, then MiniMax when configured; keep local Ollama models within the laptop-safe <=8B class and route larger or contended semantic work to MiniMax. -Print it once, then continue — do not wait for the user to supply a key. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. +MiniMax is the token-plan fallback, not the default workhorse. Graphify starts semantic chunks on local Ollama and spills only a capped fraction to MiniMax when local chunks are slow, fail through the local model chain, or laptop CPU/GPU load is high. Tune with `GRAPHIFY_OLLAMA_BALANCE=auto|local|remote|defer`, `GRAPHIFY_OLLAMA_FALLBACK_MODELS=qwen2.5-coder:3b,gemma3:4b`, and `GRAPHIFY_OLLAMA_MINIMAX_MAX_FRACTION` (default 0.25). Set `GRAPHIFY_DISABLE_MINIMAX_FALLBACK=1` for strict local-only runs. NVIDIA NIM is explicit-only (`--backend nim`) and is not part of the automatic graphify path. -> **No other API keys are read.** When `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, semantic extraction falls to the host agent itself — the running session is the LLM. On a host that dispatches subagents (e.g. Claude Code), dispatch them as written in Part B. On a host that runs the CLI directly in a terminal and cannot dispatch subagents, do not stall: a code-only corpus has no semantic work, so write the empty semantic file (Part B "Fast path") and continue to Part C; for a corpus with docs/papers/images, either set a Gemini key or extract those inline yourself, but in no case prompt for `ANTHROPIC_API_KEY` — that prompt is a misread of this skill. +For workflow-needed indexing, run the AST path immediately (`graphify update .`) and let semantic extraction run opportunistically. Large semantic rebuilds that can wait should use `GRAPHIFY_OLLAMA_BALANCE=defer` during the day; graphify records the queued night rebuild hint in `graphify-out/semantic-rebuild-queue.jsonl`. Prefer 20:00-06:00 for heavy local rebuilds, with 03:00-06:00 as the safest window. + +If neither local Ollama nor MiniMax can run, print this one-liner to the user, then continue with the host-model subagent dispatch in Part B: +> Tip: start Ollama with `qwen2.5-coder:3b` for local <=8B-class semantic extraction, or set `MINIMAX_API_KEY` in `~/.graphify/credentials.json` for MiniMax fallback (`pip install 'graphifyy[ollama,minimax]'`). + +> **No other API keys are read by the automatic skill path.** If Ollama/MiniMax are unavailable, fall straight through to host subagent dispatch (Part B below) — the host session itself is the LLM. If a host agent prompts the user for `ANTHROPIC_API_KEY` to run extraction, that prompt is a misread of this skill — ignore it and dispatch subagents as written. **Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** diff --git a/tools/skillgen/expected/graphify__skill-opencode.md b/tools/skillgen/expected/graphify__skill-opencode.md index 8c3ce3eed..277328e2f 100644 --- a/tools/skillgen/expected/graphify__skill-opencode.md +++ b/tools/skillgen/expected/graphify__skill-opencode.md @@ -151,14 +151,18 @@ Skip this step entirely if `detect` returned zero `video` files. When the corpus This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so it needs nothing here: go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) uses Gemini **only if** `GEMINI_API_KEY`/`GOOGLE_API_KEY` is already set; otherwise the host agent itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you catch yourself about to prompt for, wait on, or stop because of a missing API key, that is a misread of this skill — proceed without one. +> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so go straight to Part A and skip Part B. Semantic extraction starts on local Ollama when available and can fall back to MiniMax only when configured. If you cannot dispatch subagents and neither local Ollama nor MiniMax can run, do not stall: a code-only corpus has no semantic work (write the empty semantic file and continue to Part C); for docs/papers/images, extract them inline yourself. If you catch yourself about to prompt for or block on a missing API key, that is a misread of this skill — proceed without one. -**Before semantic extraction:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: -> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). +**Before dispatching subagents:** prefer graphify's direct local backend path. Use `graphify.llm.extract_corpus_parallel(files, backend="ollama", allow_minimax_fallback=True, max_concurrency=1)` for semantic extraction when local Ollama is available. The default fallback order is local `qwen2.5-coder:3b`, then local `gemma3:4b`, then MiniMax when configured; keep local Ollama models within the laptop-safe <=8B class and route larger or contended semantic work to MiniMax. -Print it once, then continue — do not wait for the user to supply a key. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. +MiniMax is the token-plan fallback, not the default workhorse. Graphify starts semantic chunks on local Ollama and spills only a capped fraction to MiniMax when local chunks are slow, fail through the local model chain, or laptop CPU/GPU load is high. Tune with `GRAPHIFY_OLLAMA_BALANCE=auto|local|remote|defer`, `GRAPHIFY_OLLAMA_FALLBACK_MODELS=qwen2.5-coder:3b,gemma3:4b`, and `GRAPHIFY_OLLAMA_MINIMAX_MAX_FRACTION` (default 0.25). Set `GRAPHIFY_DISABLE_MINIMAX_FALLBACK=1` for strict local-only runs. NVIDIA NIM is explicit-only (`--backend nim`) and is not part of the automatic graphify path. -> **No other API keys are read.** When `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, semantic extraction falls to the host agent itself — the running session is the LLM. On a host that dispatches subagents (e.g. Claude Code), dispatch them as written in Part B. On a host that runs the CLI directly in a terminal and cannot dispatch subagents, do not stall: a code-only corpus has no semantic work, so write the empty semantic file (Part B "Fast path") and continue to Part C; for a corpus with docs/papers/images, either set a Gemini key or extract those inline yourself, but in no case prompt for `ANTHROPIC_API_KEY` — that prompt is a misread of this skill. +For workflow-needed indexing, run the AST path immediately (`graphify update .`) and let semantic extraction run opportunistically. Large semantic rebuilds that can wait should use `GRAPHIFY_OLLAMA_BALANCE=defer` during the day; graphify records the queued night rebuild hint in `graphify-out/semantic-rebuild-queue.jsonl`. Prefer 20:00-06:00 for heavy local rebuilds, with 03:00-06:00 as the safest window. + +If neither local Ollama nor MiniMax can run, print this one-liner to the user, then continue with the host-model subagent dispatch in Part B: +> Tip: start Ollama with `qwen2.5-coder:3b` for local <=8B-class semantic extraction, or set `MINIMAX_API_KEY` in `~/.graphify/credentials.json` for MiniMax fallback (`pip install 'graphifyy[ollama,minimax]'`). + +> **No other API keys are read by the automatic skill path.** If Ollama/MiniMax are unavailable, fall straight through to host subagent dispatch (Part B below) — the host session itself is the LLM. If a host agent prompts the user for `ANTHROPIC_API_KEY` to run extraction, that prompt is a misread of this skill — ignore it and dispatch subagents as written. **Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** diff --git a/tools/skillgen/expected/graphify__skill-pi.md b/tools/skillgen/expected/graphify__skill-pi.md index bafbca074..7201db50a 100644 --- a/tools/skillgen/expected/graphify__skill-pi.md +++ b/tools/skillgen/expected/graphify__skill-pi.md @@ -151,14 +151,18 @@ Skip this step entirely if `detect` returned zero `video` files. When the corpus This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so it needs nothing here: go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) uses Gemini **only if** `GEMINI_API_KEY`/`GOOGLE_API_KEY` is already set; otherwise the host agent itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you catch yourself about to prompt for, wait on, or stop because of a missing API key, that is a misread of this skill — proceed without one. +> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so go straight to Part A and skip Part B. Semantic extraction starts on local Ollama when available and can fall back to MiniMax only when configured. If you cannot dispatch subagents and neither local Ollama nor MiniMax can run, do not stall: a code-only corpus has no semantic work (write the empty semantic file and continue to Part C); for docs/papers/images, extract them inline yourself. If you catch yourself about to prompt for or block on a missing API key, that is a misread of this skill — proceed without one. -**Before semantic extraction:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: -> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). +**Before dispatching subagents:** prefer graphify's direct local backend path. Use `graphify.llm.extract_corpus_parallel(files, backend="ollama", allow_minimax_fallback=True, max_concurrency=1)` for semantic extraction when local Ollama is available. The default fallback order is local `qwen2.5-coder:3b`, then local `gemma3:4b`, then MiniMax when configured; keep local Ollama models within the laptop-safe <=8B class and route larger or contended semantic work to MiniMax. -Print it once, then continue — do not wait for the user to supply a key. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. +MiniMax is the token-plan fallback, not the default workhorse. Graphify starts semantic chunks on local Ollama and spills only a capped fraction to MiniMax when local chunks are slow, fail through the local model chain, or laptop CPU/GPU load is high. Tune with `GRAPHIFY_OLLAMA_BALANCE=auto|local|remote|defer`, `GRAPHIFY_OLLAMA_FALLBACK_MODELS=qwen2.5-coder:3b,gemma3:4b`, and `GRAPHIFY_OLLAMA_MINIMAX_MAX_FRACTION` (default 0.25). Set `GRAPHIFY_DISABLE_MINIMAX_FALLBACK=1` for strict local-only runs. NVIDIA NIM is explicit-only (`--backend nim`) and is not part of the automatic graphify path. -> **No other API keys are read.** When `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, semantic extraction falls to the host agent itself — the running session is the LLM. On a host that dispatches subagents (e.g. Claude Code), dispatch them as written in Part B. On a host that runs the CLI directly in a terminal and cannot dispatch subagents, do not stall: a code-only corpus has no semantic work, so write the empty semantic file (Part B "Fast path") and continue to Part C; for a corpus with docs/papers/images, either set a Gemini key or extract those inline yourself, but in no case prompt for `ANTHROPIC_API_KEY` — that prompt is a misread of this skill. +For workflow-needed indexing, run the AST path immediately (`graphify update .`) and let semantic extraction run opportunistically. Large semantic rebuilds that can wait should use `GRAPHIFY_OLLAMA_BALANCE=defer` during the day; graphify records the queued night rebuild hint in `graphify-out/semantic-rebuild-queue.jsonl`. Prefer 20:00-06:00 for heavy local rebuilds, with 03:00-06:00 as the safest window. + +If neither local Ollama nor MiniMax can run, print this one-liner to the user, then continue with the host-model subagent dispatch in Part B: +> Tip: start Ollama with `qwen2.5-coder:3b` for local <=8B-class semantic extraction, or set `MINIMAX_API_KEY` in `~/.graphify/credentials.json` for MiniMax fallback (`pip install 'graphifyy[ollama,minimax]'`). + +> **No other API keys are read by the automatic skill path.** If Ollama/MiniMax are unavailable, fall straight through to host subagent dispatch (Part B below) — the host session itself is the LLM. If a host agent prompts the user for `ANTHROPIC_API_KEY` to run extraction, that prompt is a misread of this skill — ignore it and dispatch subagents as written. **Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** diff --git a/tools/skillgen/expected/graphify__skill-trae.md b/tools/skillgen/expected/graphify__skill-trae.md index af59166ae..d033e2dda 100644 --- a/tools/skillgen/expected/graphify__skill-trae.md +++ b/tools/skillgen/expected/graphify__skill-trae.md @@ -151,14 +151,18 @@ Skip this step entirely if `detect` returned zero `video` files. When the corpus This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so it needs nothing here: go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) uses Gemini **only if** `GEMINI_API_KEY`/`GOOGLE_API_KEY` is already set; otherwise the host agent itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you catch yourself about to prompt for, wait on, or stop because of a missing API key, that is a misread of this skill — proceed without one. +> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so go straight to Part A and skip Part B. Semantic extraction starts on local Ollama when available and can fall back to MiniMax only when configured. If you cannot dispatch subagents and neither local Ollama nor MiniMax can run, do not stall: a code-only corpus has no semantic work (write the empty semantic file and continue to Part C); for docs/papers/images, extract them inline yourself. If you catch yourself about to prompt for or block on a missing API key, that is a misread of this skill — proceed without one. -**Before semantic extraction:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: -> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). +**Before dispatching subagents:** prefer graphify's direct local backend path. Use `graphify.llm.extract_corpus_parallel(files, backend="ollama", allow_minimax_fallback=True, max_concurrency=1)` for semantic extraction when local Ollama is available. The default fallback order is local `qwen2.5-coder:3b`, then local `gemma3:4b`, then MiniMax when configured; keep local Ollama models within the laptop-safe <=8B class and route larger or contended semantic work to MiniMax. -Print it once, then continue — do not wait for the user to supply a key. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. +MiniMax is the token-plan fallback, not the default workhorse. Graphify starts semantic chunks on local Ollama and spills only a capped fraction to MiniMax when local chunks are slow, fail through the local model chain, or laptop CPU/GPU load is high. Tune with `GRAPHIFY_OLLAMA_BALANCE=auto|local|remote|defer`, `GRAPHIFY_OLLAMA_FALLBACK_MODELS=qwen2.5-coder:3b,gemma3:4b`, and `GRAPHIFY_OLLAMA_MINIMAX_MAX_FRACTION` (default 0.25). Set `GRAPHIFY_DISABLE_MINIMAX_FALLBACK=1` for strict local-only runs. NVIDIA NIM is explicit-only (`--backend nim`) and is not part of the automatic graphify path. -> **No other API keys are read.** When `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, semantic extraction falls to the host agent itself — the running session is the LLM. On a host that dispatches subagents (e.g. Claude Code), dispatch them as written in Part B. On a host that runs the CLI directly in a terminal and cannot dispatch subagents, do not stall: a code-only corpus has no semantic work, so write the empty semantic file (Part B "Fast path") and continue to Part C; for a corpus with docs/papers/images, either set a Gemini key or extract those inline yourself, but in no case prompt for `ANTHROPIC_API_KEY` — that prompt is a misread of this skill. +For workflow-needed indexing, run the AST path immediately (`graphify update .`) and let semantic extraction run opportunistically. Large semantic rebuilds that can wait should use `GRAPHIFY_OLLAMA_BALANCE=defer` during the day; graphify records the queued night rebuild hint in `graphify-out/semantic-rebuild-queue.jsonl`. Prefer 20:00-06:00 for heavy local rebuilds, with 03:00-06:00 as the safest window. + +If neither local Ollama nor MiniMax can run, print this one-liner to the user, then continue with the host-model subagent dispatch in Part B: +> Tip: start Ollama with `qwen2.5-coder:3b` for local <=8B-class semantic extraction, or set `MINIMAX_API_KEY` in `~/.graphify/credentials.json` for MiniMax fallback (`pip install 'graphifyy[ollama,minimax]'`). + +> **No other API keys are read by the automatic skill path.** If Ollama/MiniMax are unavailable, fall straight through to host subagent dispatch (Part B below) — the host session itself is the LLM. If a host agent prompts the user for `ANTHROPIC_API_KEY` to run extraction, that prompt is a misread of this skill — ignore it and dispatch subagents as written. **Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** diff --git a/tools/skillgen/expected/graphify__skill-vscode.md b/tools/skillgen/expected/graphify__skill-vscode.md index 11a2837b5..07c4dd472 100644 --- a/tools/skillgen/expected/graphify__skill-vscode.md +++ b/tools/skillgen/expected/graphify__skill-vscode.md @@ -151,14 +151,18 @@ Skip this step entirely if `detect` returned zero `video` files. When the corpus This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so it needs nothing here: go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) uses Gemini **only if** `GEMINI_API_KEY`/`GOOGLE_API_KEY` is already set; otherwise the host agent itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you catch yourself about to prompt for, wait on, or stop because of a missing API key, that is a misread of this skill — proceed without one. +> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so go straight to Part A and skip Part B. Semantic extraction starts on local Ollama when available and can fall back to MiniMax only when configured. If you cannot dispatch subagents and neither local Ollama nor MiniMax can run, do not stall: a code-only corpus has no semantic work (write the empty semantic file and continue to Part C); for docs/papers/images, extract them inline yourself. If you catch yourself about to prompt for or block on a missing API key, that is a misread of this skill — proceed without one. -**Before semantic extraction:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: -> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). +**Before dispatching subagents:** prefer graphify's direct local backend path. Use `graphify.llm.extract_corpus_parallel(files, backend="ollama", allow_minimax_fallback=True, max_concurrency=1)` for semantic extraction when local Ollama is available. The default fallback order is local `qwen2.5-coder:3b`, then local `gemma3:4b`, then MiniMax when configured; keep local Ollama models within the laptop-safe <=8B class and route larger or contended semantic work to MiniMax. -Print it once, then continue — do not wait for the user to supply a key. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. +MiniMax is the token-plan fallback, not the default workhorse. Graphify starts semantic chunks on local Ollama and spills only a capped fraction to MiniMax when local chunks are slow, fail through the local model chain, or laptop CPU/GPU load is high. Tune with `GRAPHIFY_OLLAMA_BALANCE=auto|local|remote|defer`, `GRAPHIFY_OLLAMA_FALLBACK_MODELS=qwen2.5-coder:3b,gemma3:4b`, and `GRAPHIFY_OLLAMA_MINIMAX_MAX_FRACTION` (default 0.25). Set `GRAPHIFY_DISABLE_MINIMAX_FALLBACK=1` for strict local-only runs. NVIDIA NIM is explicit-only (`--backend nim`) and is not part of the automatic graphify path. -> **No other API keys are read.** When `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, semantic extraction falls to the host agent itself — the running session is the LLM. On a host that dispatches subagents (e.g. Claude Code), dispatch them as written in Part B. On a host that runs the CLI directly in a terminal and cannot dispatch subagents, do not stall: a code-only corpus has no semantic work, so write the empty semantic file (Part B "Fast path") and continue to Part C; for a corpus with docs/papers/images, either set a Gemini key or extract those inline yourself, but in no case prompt for `ANTHROPIC_API_KEY` — that prompt is a misread of this skill. +For workflow-needed indexing, run the AST path immediately (`graphify update .`) and let semantic extraction run opportunistically. Large semantic rebuilds that can wait should use `GRAPHIFY_OLLAMA_BALANCE=defer` during the day; graphify records the queued night rebuild hint in `graphify-out/semantic-rebuild-queue.jsonl`. Prefer 20:00-06:00 for heavy local rebuilds, with 03:00-06:00 as the safest window. + +If neither local Ollama nor MiniMax can run, print this one-liner to the user, then continue with the host-model subagent dispatch in Part B: +> Tip: start Ollama with `qwen2.5-coder:3b` for local <=8B-class semantic extraction, or set `MINIMAX_API_KEY` in `~/.graphify/credentials.json` for MiniMax fallback (`pip install 'graphifyy[ollama,minimax]'`). + +> **No other API keys are read by the automatic skill path.** If Ollama/MiniMax are unavailable, fall straight through to host subagent dispatch (Part B below) — the host session itself is the LLM. If a host agent prompts the user for `ANTHROPIC_API_KEY` to run extraction, that prompt is a misread of this skill — ignore it and dispatch subagents as written. **Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** diff --git a/tools/skillgen/expected/graphify__skill-windows.md b/tools/skillgen/expected/graphify__skill-windows.md index 0dc73947a..26a091837 100644 --- a/tools/skillgen/expected/graphify__skill-windows.md +++ b/tools/skillgen/expected/graphify__skill-windows.md @@ -173,14 +173,18 @@ Skip this step entirely if `detect` returned zero `video` files. When the corpus This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so it needs nothing here: go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) uses Gemini **only if** `GEMINI_API_KEY`/`GOOGLE_API_KEY` is already set; otherwise the host agent itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you catch yourself about to prompt for, wait on, or stop because of a missing API key, that is a misread of this skill — proceed without one. +> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so go straight to Part A and skip Part B. Semantic extraction starts on local Ollama when available and can fall back to MiniMax only when configured. If you cannot dispatch subagents and neither local Ollama nor MiniMax can run, do not stall: a code-only corpus has no semantic work (write the empty semantic file and continue to Part C); for docs/papers/images, extract them inline yourself. If you catch yourself about to prompt for or block on a missing API key, that is a misread of this skill — proceed without one. -**Before semantic extraction:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: -> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). +**Before dispatching subagents:** prefer graphify's direct local backend path. Use `graphify.llm.extract_corpus_parallel(files, backend="ollama", allow_minimax_fallback=True, max_concurrency=1)` for semantic extraction when local Ollama is available. The default fallback order is local `qwen2.5-coder:3b`, then local `gemma3:4b`, then MiniMax when configured; keep local Ollama models within the laptop-safe <=8B class and route larger or contended semantic work to MiniMax. -Print it once, then continue — do not wait for the user to supply a key. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. +MiniMax is the token-plan fallback, not the default workhorse. Graphify starts semantic chunks on local Ollama and spills only a capped fraction to MiniMax when local chunks are slow, fail through the local model chain, or laptop CPU/GPU load is high. Tune with `GRAPHIFY_OLLAMA_BALANCE=auto|local|remote|defer`, `GRAPHIFY_OLLAMA_FALLBACK_MODELS=qwen2.5-coder:3b,gemma3:4b`, and `GRAPHIFY_OLLAMA_MINIMAX_MAX_FRACTION` (default 0.25). Set `GRAPHIFY_DISABLE_MINIMAX_FALLBACK=1` for strict local-only runs. NVIDIA NIM is explicit-only (`--backend nim`) and is not part of the automatic graphify path. -> **No other API keys are read.** When `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, semantic extraction falls to the host agent itself — the running session is the LLM. On a host that dispatches subagents (e.g. Claude Code), dispatch them as written in Part B. On a host that runs the CLI directly in a terminal and cannot dispatch subagents, do not stall: a code-only corpus has no semantic work, so write the empty semantic file (Part B "Fast path") and continue to Part C; for a corpus with docs/papers/images, either set a Gemini key or extract those inline yourself, but in no case prompt for `ANTHROPIC_API_KEY` — that prompt is a misread of this skill. +For workflow-needed indexing, run the AST path immediately (`graphify update .`) and let semantic extraction run opportunistically. Large semantic rebuilds that can wait should use `GRAPHIFY_OLLAMA_BALANCE=defer` during the day; graphify records the queued night rebuild hint in `graphify-out/semantic-rebuild-queue.jsonl`. Prefer 20:00-06:00 for heavy local rebuilds, with 03:00-06:00 as the safest window. + +If neither local Ollama nor MiniMax can run, print this one-liner to the user, then continue with the host-model subagent dispatch in Part B: +> Tip: start Ollama with `qwen2.5-coder:3b` for local <=8B-class semantic extraction, or set `MINIMAX_API_KEY` in `~/.graphify/credentials.json` for MiniMax fallback (`pip install 'graphifyy[ollama,minimax]'`). + +> **No other API keys are read by the automatic skill path.** If Ollama/MiniMax are unavailable, fall straight through to host subagent dispatch (Part B below) — the host session itself is the LLM. If a host agent prompts the user for `ANTHROPIC_API_KEY` to run extraction, that prompt is a misread of this skill — ignore it and dispatch subagents as written. **Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** diff --git a/tools/skillgen/expected/graphify__skill.md b/tools/skillgen/expected/graphify__skill.md index bafbca074..7201db50a 100644 --- a/tools/skillgen/expected/graphify__skill.md +++ b/tools/skillgen/expected/graphify__skill.md @@ -151,14 +151,18 @@ Skip this step entirely if `detect` returned zero `video` files. When the corpus This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so it needs nothing here: go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) uses Gemini **only if** `GEMINI_API_KEY`/`GOOGLE_API_KEY` is already set; otherwise the host agent itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you catch yourself about to prompt for, wait on, or stop because of a missing API key, that is a misread of this skill — proceed without one. +> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so go straight to Part A and skip Part B. Semantic extraction starts on local Ollama when available and can fall back to MiniMax only when configured. If you cannot dispatch subagents and neither local Ollama nor MiniMax can run, do not stall: a code-only corpus has no semantic work (write the empty semantic file and continue to Part C); for docs/papers/images, extract them inline yourself. If you catch yourself about to prompt for or block on a missing API key, that is a misread of this skill — proceed without one. -**Before semantic extraction:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: -> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). +**Before dispatching subagents:** prefer graphify's direct local backend path. Use `graphify.llm.extract_corpus_parallel(files, backend="ollama", allow_minimax_fallback=True, max_concurrency=1)` for semantic extraction when local Ollama is available. The default fallback order is local `qwen2.5-coder:3b`, then local `gemma3:4b`, then MiniMax when configured; keep local Ollama models within the laptop-safe <=8B class and route larger or contended semantic work to MiniMax. -Print it once, then continue — do not wait for the user to supply a key. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. +MiniMax is the token-plan fallback, not the default workhorse. Graphify starts semantic chunks on local Ollama and spills only a capped fraction to MiniMax when local chunks are slow, fail through the local model chain, or laptop CPU/GPU load is high. Tune with `GRAPHIFY_OLLAMA_BALANCE=auto|local|remote|defer`, `GRAPHIFY_OLLAMA_FALLBACK_MODELS=qwen2.5-coder:3b,gemma3:4b`, and `GRAPHIFY_OLLAMA_MINIMAX_MAX_FRACTION` (default 0.25). Set `GRAPHIFY_DISABLE_MINIMAX_FALLBACK=1` for strict local-only runs. NVIDIA NIM is explicit-only (`--backend nim`) and is not part of the automatic graphify path. -> **No other API keys are read.** When `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, semantic extraction falls to the host agent itself — the running session is the LLM. On a host that dispatches subagents (e.g. Claude Code), dispatch them as written in Part B. On a host that runs the CLI directly in a terminal and cannot dispatch subagents, do not stall: a code-only corpus has no semantic work, so write the empty semantic file (Part B "Fast path") and continue to Part C; for a corpus with docs/papers/images, either set a Gemini key or extract those inline yourself, but in no case prompt for `ANTHROPIC_API_KEY` — that prompt is a misread of this skill. +For workflow-needed indexing, run the AST path immediately (`graphify update .`) and let semantic extraction run opportunistically. Large semantic rebuilds that can wait should use `GRAPHIFY_OLLAMA_BALANCE=defer` during the day; graphify records the queued night rebuild hint in `graphify-out/semantic-rebuild-queue.jsonl`. Prefer 20:00-06:00 for heavy local rebuilds, with 03:00-06:00 as the safest window. + +If neither local Ollama nor MiniMax can run, print this one-liner to the user, then continue with the host-model subagent dispatch in Part B: +> Tip: start Ollama with `qwen2.5-coder:3b` for local <=8B-class semantic extraction, or set `MINIMAX_API_KEY` in `~/.graphify/credentials.json` for MiniMax fallback (`pip install 'graphifyy[ollama,minimax]'`). + +> **No other API keys are read by the automatic skill path.** If Ollama/MiniMax are unavailable, fall straight through to host subagent dispatch (Part B below) — the host session itself is the LLM. If a host agent prompts the user for `ANTHROPIC_API_KEY` to run extraction, that prompt is a misread of this skill — ignore it and dispatch subagents as written. **Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** diff --git a/tools/skillgen/expected/graphify__skills__agents__references__github-and-merge.md b/tools/skillgen/expected/graphify__skills__agents__references__github-and-merge.md index a41ea06e1..48b4297b8 100644 --- a/tools/skillgen/expected/graphify__skills__agents__references__github-and-merge.md +++ b/tools/skillgen/expected/graphify__skills__agents__references__github-and-merge.md @@ -33,7 +33,7 @@ The skill pipeline writes all intermediate and final outputs to `graphify-out/` graphify extract ./core/ # → ./core/graphify-out/graph.json graphify extract ./service/ # → ./service/graphify-out/graph.json graphify extract ./platform/ # → ./platform/graphify-out/graph.json -# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set +# Add --backend minimax|nim|gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set # Then merge at the project root: graphify merge-graphs \ diff --git a/tools/skillgen/expected/graphify__skills__agents__references__update.md b/tools/skillgen/expected/graphify__skills__agents__references__update.md index fa2612180..cd7c2d015 100644 --- a/tools/skillgen/expected/graphify__skills__agents__references__update.md +++ b/tools/skillgen/expected/graphify__skills__agents__references__update.md @@ -6,6 +6,8 @@ Load this only when the user passed `--update` or `--cluster-only`. A first-time Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +For very large repos under `/media/naray/backup_np_2/github/`, do not run semantic `--update` repeatedly during the day. Let automatic AST indexing keep code workflows fresh, and reserve this full update path for an explicit user request or one daily night-window refresh after 20:00 (03:00-06:00 safest). If `graphify-out/nightly-update-hint.json` exists, treat it as the queue hint for that daily refresh. + ```bash $(cat graphify-out/.graphify_python) -c " import sys, json diff --git a/tools/skillgen/expected/graphify__skills__amp__references__github-and-merge.md b/tools/skillgen/expected/graphify__skills__amp__references__github-and-merge.md index a41ea06e1..48b4297b8 100644 --- a/tools/skillgen/expected/graphify__skills__amp__references__github-and-merge.md +++ b/tools/skillgen/expected/graphify__skills__amp__references__github-and-merge.md @@ -33,7 +33,7 @@ The skill pipeline writes all intermediate and final outputs to `graphify-out/` graphify extract ./core/ # → ./core/graphify-out/graph.json graphify extract ./service/ # → ./service/graphify-out/graph.json graphify extract ./platform/ # → ./platform/graphify-out/graph.json -# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set +# Add --backend minimax|nim|gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set # Then merge at the project root: graphify merge-graphs \ diff --git a/tools/skillgen/expected/graphify__skills__amp__references__update.md b/tools/skillgen/expected/graphify__skills__amp__references__update.md index fa2612180..cd7c2d015 100644 --- a/tools/skillgen/expected/graphify__skills__amp__references__update.md +++ b/tools/skillgen/expected/graphify__skills__amp__references__update.md @@ -6,6 +6,8 @@ Load this only when the user passed `--update` or `--cluster-only`. A first-time Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +For very large repos under `/media/naray/backup_np_2/github/`, do not run semantic `--update` repeatedly during the day. Let automatic AST indexing keep code workflows fresh, and reserve this full update path for an explicit user request or one daily night-window refresh after 20:00 (03:00-06:00 safest). If `graphify-out/nightly-update-hint.json` exists, treat it as the queue hint for that daily refresh. + ```bash $(cat graphify-out/.graphify_python) -c " import sys, json diff --git a/tools/skillgen/expected/graphify__skills__claude__references__github-and-merge.md b/tools/skillgen/expected/graphify__skills__claude__references__github-and-merge.md index a41ea06e1..48b4297b8 100644 --- a/tools/skillgen/expected/graphify__skills__claude__references__github-and-merge.md +++ b/tools/skillgen/expected/graphify__skills__claude__references__github-and-merge.md @@ -33,7 +33,7 @@ The skill pipeline writes all intermediate and final outputs to `graphify-out/` graphify extract ./core/ # → ./core/graphify-out/graph.json graphify extract ./service/ # → ./service/graphify-out/graph.json graphify extract ./platform/ # → ./platform/graphify-out/graph.json -# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set +# Add --backend minimax|nim|gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set # Then merge at the project root: graphify merge-graphs \ diff --git a/tools/skillgen/expected/graphify__skills__claude__references__update.md b/tools/skillgen/expected/graphify__skills__claude__references__update.md index fa2612180..cd7c2d015 100644 --- a/tools/skillgen/expected/graphify__skills__claude__references__update.md +++ b/tools/skillgen/expected/graphify__skills__claude__references__update.md @@ -6,6 +6,8 @@ Load this only when the user passed `--update` or `--cluster-only`. A first-time Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +For very large repos under `/media/naray/backup_np_2/github/`, do not run semantic `--update` repeatedly during the day. Let automatic AST indexing keep code workflows fresh, and reserve this full update path for an explicit user request or one daily night-window refresh after 20:00 (03:00-06:00 safest). If `graphify-out/nightly-update-hint.json` exists, treat it as the queue hint for that daily refresh. + ```bash $(cat graphify-out/.graphify_python) -c " import sys, json diff --git a/tools/skillgen/expected/graphify__skills__claw__references__github-and-merge.md b/tools/skillgen/expected/graphify__skills__claw__references__github-and-merge.md index a41ea06e1..48b4297b8 100644 --- a/tools/skillgen/expected/graphify__skills__claw__references__github-and-merge.md +++ b/tools/skillgen/expected/graphify__skills__claw__references__github-and-merge.md @@ -33,7 +33,7 @@ The skill pipeline writes all intermediate and final outputs to `graphify-out/` graphify extract ./core/ # → ./core/graphify-out/graph.json graphify extract ./service/ # → ./service/graphify-out/graph.json graphify extract ./platform/ # → ./platform/graphify-out/graph.json -# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set +# Add --backend minimax|nim|gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set # Then merge at the project root: graphify merge-graphs \ diff --git a/tools/skillgen/expected/graphify__skills__claw__references__update.md b/tools/skillgen/expected/graphify__skills__claw__references__update.md index fa2612180..cd7c2d015 100644 --- a/tools/skillgen/expected/graphify__skills__claw__references__update.md +++ b/tools/skillgen/expected/graphify__skills__claw__references__update.md @@ -6,6 +6,8 @@ Load this only when the user passed `--update` or `--cluster-only`. A first-time Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +For very large repos under `/media/naray/backup_np_2/github/`, do not run semantic `--update` repeatedly during the day. Let automatic AST indexing keep code workflows fresh, and reserve this full update path for an explicit user request or one daily night-window refresh after 20:00 (03:00-06:00 safest). If `graphify-out/nightly-update-hint.json` exists, treat it as the queue hint for that daily refresh. + ```bash $(cat graphify-out/.graphify_python) -c " import sys, json diff --git a/tools/skillgen/expected/graphify__skills__codex__references__github-and-merge.md b/tools/skillgen/expected/graphify__skills__codex__references__github-and-merge.md index a41ea06e1..48b4297b8 100644 --- a/tools/skillgen/expected/graphify__skills__codex__references__github-and-merge.md +++ b/tools/skillgen/expected/graphify__skills__codex__references__github-and-merge.md @@ -33,7 +33,7 @@ The skill pipeline writes all intermediate and final outputs to `graphify-out/` graphify extract ./core/ # → ./core/graphify-out/graph.json graphify extract ./service/ # → ./service/graphify-out/graph.json graphify extract ./platform/ # → ./platform/graphify-out/graph.json -# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set +# Add --backend minimax|nim|gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set # Then merge at the project root: graphify merge-graphs \ diff --git a/tools/skillgen/expected/graphify__skills__codex__references__update.md b/tools/skillgen/expected/graphify__skills__codex__references__update.md index fa2612180..cd7c2d015 100644 --- a/tools/skillgen/expected/graphify__skills__codex__references__update.md +++ b/tools/skillgen/expected/graphify__skills__codex__references__update.md @@ -6,6 +6,8 @@ Load this only when the user passed `--update` or `--cluster-only`. A first-time Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +For very large repos under `/media/naray/backup_np_2/github/`, do not run semantic `--update` repeatedly during the day. Let automatic AST indexing keep code workflows fresh, and reserve this full update path for an explicit user request or one daily night-window refresh after 20:00 (03:00-06:00 safest). If `graphify-out/nightly-update-hint.json` exists, treat it as the queue hint for that daily refresh. + ```bash $(cat graphify-out/.graphify_python) -c " import sys, json diff --git a/tools/skillgen/expected/graphify__skills__copilot__references__github-and-merge.md b/tools/skillgen/expected/graphify__skills__copilot__references__github-and-merge.md index a41ea06e1..48b4297b8 100644 --- a/tools/skillgen/expected/graphify__skills__copilot__references__github-and-merge.md +++ b/tools/skillgen/expected/graphify__skills__copilot__references__github-and-merge.md @@ -33,7 +33,7 @@ The skill pipeline writes all intermediate and final outputs to `graphify-out/` graphify extract ./core/ # → ./core/graphify-out/graph.json graphify extract ./service/ # → ./service/graphify-out/graph.json graphify extract ./platform/ # → ./platform/graphify-out/graph.json -# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set +# Add --backend minimax|nim|gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set # Then merge at the project root: graphify merge-graphs \ diff --git a/tools/skillgen/expected/graphify__skills__copilot__references__update.md b/tools/skillgen/expected/graphify__skills__copilot__references__update.md index fa2612180..cd7c2d015 100644 --- a/tools/skillgen/expected/graphify__skills__copilot__references__update.md +++ b/tools/skillgen/expected/graphify__skills__copilot__references__update.md @@ -6,6 +6,8 @@ Load this only when the user passed `--update` or `--cluster-only`. A first-time Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +For very large repos under `/media/naray/backup_np_2/github/`, do not run semantic `--update` repeatedly during the day. Let automatic AST indexing keep code workflows fresh, and reserve this full update path for an explicit user request or one daily night-window refresh after 20:00 (03:00-06:00 safest). If `graphify-out/nightly-update-hint.json` exists, treat it as the queue hint for that daily refresh. + ```bash $(cat graphify-out/.graphify_python) -c " import sys, json diff --git a/tools/skillgen/expected/graphify__skills__droid__references__github-and-merge.md b/tools/skillgen/expected/graphify__skills__droid__references__github-and-merge.md index a41ea06e1..48b4297b8 100644 --- a/tools/skillgen/expected/graphify__skills__droid__references__github-and-merge.md +++ b/tools/skillgen/expected/graphify__skills__droid__references__github-and-merge.md @@ -33,7 +33,7 @@ The skill pipeline writes all intermediate and final outputs to `graphify-out/` graphify extract ./core/ # → ./core/graphify-out/graph.json graphify extract ./service/ # → ./service/graphify-out/graph.json graphify extract ./platform/ # → ./platform/graphify-out/graph.json -# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set +# Add --backend minimax|nim|gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set # Then merge at the project root: graphify merge-graphs \ diff --git a/tools/skillgen/expected/graphify__skills__droid__references__update.md b/tools/skillgen/expected/graphify__skills__droid__references__update.md index fa2612180..cd7c2d015 100644 --- a/tools/skillgen/expected/graphify__skills__droid__references__update.md +++ b/tools/skillgen/expected/graphify__skills__droid__references__update.md @@ -6,6 +6,8 @@ Load this only when the user passed `--update` or `--cluster-only`. A first-time Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +For very large repos under `/media/naray/backup_np_2/github/`, do not run semantic `--update` repeatedly during the day. Let automatic AST indexing keep code workflows fresh, and reserve this full update path for an explicit user request or one daily night-window refresh after 20:00 (03:00-06:00 safest). If `graphify-out/nightly-update-hint.json` exists, treat it as the queue hint for that daily refresh. + ```bash $(cat graphify-out/.graphify_python) -c " import sys, json diff --git a/tools/skillgen/expected/graphify__skills__kilo__references__github-and-merge.md b/tools/skillgen/expected/graphify__skills__kilo__references__github-and-merge.md index a41ea06e1..48b4297b8 100644 --- a/tools/skillgen/expected/graphify__skills__kilo__references__github-and-merge.md +++ b/tools/skillgen/expected/graphify__skills__kilo__references__github-and-merge.md @@ -33,7 +33,7 @@ The skill pipeline writes all intermediate and final outputs to `graphify-out/` graphify extract ./core/ # → ./core/graphify-out/graph.json graphify extract ./service/ # → ./service/graphify-out/graph.json graphify extract ./platform/ # → ./platform/graphify-out/graph.json -# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set +# Add --backend minimax|nim|gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set # Then merge at the project root: graphify merge-graphs \ diff --git a/tools/skillgen/expected/graphify__skills__kilo__references__update.md b/tools/skillgen/expected/graphify__skills__kilo__references__update.md index fa2612180..cd7c2d015 100644 --- a/tools/skillgen/expected/graphify__skills__kilo__references__update.md +++ b/tools/skillgen/expected/graphify__skills__kilo__references__update.md @@ -6,6 +6,8 @@ Load this only when the user passed `--update` or `--cluster-only`. A first-time Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +For very large repos under `/media/naray/backup_np_2/github/`, do not run semantic `--update` repeatedly during the day. Let automatic AST indexing keep code workflows fresh, and reserve this full update path for an explicit user request or one daily night-window refresh after 20:00 (03:00-06:00 safest). If `graphify-out/nightly-update-hint.json` exists, treat it as the queue hint for that daily refresh. + ```bash $(cat graphify-out/.graphify_python) -c " import sys, json diff --git a/tools/skillgen/expected/graphify__skills__kiro__references__github-and-merge.md b/tools/skillgen/expected/graphify__skills__kiro__references__github-and-merge.md index a41ea06e1..48b4297b8 100644 --- a/tools/skillgen/expected/graphify__skills__kiro__references__github-and-merge.md +++ b/tools/skillgen/expected/graphify__skills__kiro__references__github-and-merge.md @@ -33,7 +33,7 @@ The skill pipeline writes all intermediate and final outputs to `graphify-out/` graphify extract ./core/ # → ./core/graphify-out/graph.json graphify extract ./service/ # → ./service/graphify-out/graph.json graphify extract ./platform/ # → ./platform/graphify-out/graph.json -# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set +# Add --backend minimax|nim|gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set # Then merge at the project root: graphify merge-graphs \ diff --git a/tools/skillgen/expected/graphify__skills__kiro__references__update.md b/tools/skillgen/expected/graphify__skills__kiro__references__update.md index fa2612180..cd7c2d015 100644 --- a/tools/skillgen/expected/graphify__skills__kiro__references__update.md +++ b/tools/skillgen/expected/graphify__skills__kiro__references__update.md @@ -6,6 +6,8 @@ Load this only when the user passed `--update` or `--cluster-only`. A first-time Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +For very large repos under `/media/naray/backup_np_2/github/`, do not run semantic `--update` repeatedly during the day. Let automatic AST indexing keep code workflows fresh, and reserve this full update path for an explicit user request or one daily night-window refresh after 20:00 (03:00-06:00 safest). If `graphify-out/nightly-update-hint.json` exists, treat it as the queue hint for that daily refresh. + ```bash $(cat graphify-out/.graphify_python) -c " import sys, json diff --git a/tools/skillgen/expected/graphify__skills__opencode__references__github-and-merge.md b/tools/skillgen/expected/graphify__skills__opencode__references__github-and-merge.md index a41ea06e1..48b4297b8 100644 --- a/tools/skillgen/expected/graphify__skills__opencode__references__github-and-merge.md +++ b/tools/skillgen/expected/graphify__skills__opencode__references__github-and-merge.md @@ -33,7 +33,7 @@ The skill pipeline writes all intermediate and final outputs to `graphify-out/` graphify extract ./core/ # → ./core/graphify-out/graph.json graphify extract ./service/ # → ./service/graphify-out/graph.json graphify extract ./platform/ # → ./platform/graphify-out/graph.json -# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set +# Add --backend minimax|nim|gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set # Then merge at the project root: graphify merge-graphs \ diff --git a/tools/skillgen/expected/graphify__skills__opencode__references__update.md b/tools/skillgen/expected/graphify__skills__opencode__references__update.md index fa2612180..cd7c2d015 100644 --- a/tools/skillgen/expected/graphify__skills__opencode__references__update.md +++ b/tools/skillgen/expected/graphify__skills__opencode__references__update.md @@ -6,6 +6,8 @@ Load this only when the user passed `--update` or `--cluster-only`. A first-time Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +For very large repos under `/media/naray/backup_np_2/github/`, do not run semantic `--update` repeatedly during the day. Let automatic AST indexing keep code workflows fresh, and reserve this full update path for an explicit user request or one daily night-window refresh after 20:00 (03:00-06:00 safest). If `graphify-out/nightly-update-hint.json` exists, treat it as the queue hint for that daily refresh. + ```bash $(cat graphify-out/.graphify_python) -c " import sys, json diff --git a/tools/skillgen/expected/graphify__skills__pi__references__github-and-merge.md b/tools/skillgen/expected/graphify__skills__pi__references__github-and-merge.md index a41ea06e1..48b4297b8 100644 --- a/tools/skillgen/expected/graphify__skills__pi__references__github-and-merge.md +++ b/tools/skillgen/expected/graphify__skills__pi__references__github-and-merge.md @@ -33,7 +33,7 @@ The skill pipeline writes all intermediate and final outputs to `graphify-out/` graphify extract ./core/ # → ./core/graphify-out/graph.json graphify extract ./service/ # → ./service/graphify-out/graph.json graphify extract ./platform/ # → ./platform/graphify-out/graph.json -# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set +# Add --backend minimax|nim|gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set # Then merge at the project root: graphify merge-graphs \ diff --git a/tools/skillgen/expected/graphify__skills__pi__references__update.md b/tools/skillgen/expected/graphify__skills__pi__references__update.md index fa2612180..cd7c2d015 100644 --- a/tools/skillgen/expected/graphify__skills__pi__references__update.md +++ b/tools/skillgen/expected/graphify__skills__pi__references__update.md @@ -6,6 +6,8 @@ Load this only when the user passed `--update` or `--cluster-only`. A first-time Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +For very large repos under `/media/naray/backup_np_2/github/`, do not run semantic `--update` repeatedly during the day. Let automatic AST indexing keep code workflows fresh, and reserve this full update path for an explicit user request or one daily night-window refresh after 20:00 (03:00-06:00 safest). If `graphify-out/nightly-update-hint.json` exists, treat it as the queue hint for that daily refresh. + ```bash $(cat graphify-out/.graphify_python) -c " import sys, json diff --git a/tools/skillgen/expected/graphify__skills__trae__references__github-and-merge.md b/tools/skillgen/expected/graphify__skills__trae__references__github-and-merge.md index a41ea06e1..48b4297b8 100644 --- a/tools/skillgen/expected/graphify__skills__trae__references__github-and-merge.md +++ b/tools/skillgen/expected/graphify__skills__trae__references__github-and-merge.md @@ -33,7 +33,7 @@ The skill pipeline writes all intermediate and final outputs to `graphify-out/` graphify extract ./core/ # → ./core/graphify-out/graph.json graphify extract ./service/ # → ./service/graphify-out/graph.json graphify extract ./platform/ # → ./platform/graphify-out/graph.json -# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set +# Add --backend minimax|nim|gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set # Then merge at the project root: graphify merge-graphs \ diff --git a/tools/skillgen/expected/graphify__skills__trae__references__update.md b/tools/skillgen/expected/graphify__skills__trae__references__update.md index fa2612180..cd7c2d015 100644 --- a/tools/skillgen/expected/graphify__skills__trae__references__update.md +++ b/tools/skillgen/expected/graphify__skills__trae__references__update.md @@ -6,6 +6,8 @@ Load this only when the user passed `--update` or `--cluster-only`. A first-time Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +For very large repos under `/media/naray/backup_np_2/github/`, do not run semantic `--update` repeatedly during the day. Let automatic AST indexing keep code workflows fresh, and reserve this full update path for an explicit user request or one daily night-window refresh after 20:00 (03:00-06:00 safest). If `graphify-out/nightly-update-hint.json` exists, treat it as the queue hint for that daily refresh. + ```bash $(cat graphify-out/.graphify_python) -c " import sys, json diff --git a/tools/skillgen/expected/graphify__skills__vscode__references__github-and-merge.md b/tools/skillgen/expected/graphify__skills__vscode__references__github-and-merge.md index a41ea06e1..48b4297b8 100644 --- a/tools/skillgen/expected/graphify__skills__vscode__references__github-and-merge.md +++ b/tools/skillgen/expected/graphify__skills__vscode__references__github-and-merge.md @@ -33,7 +33,7 @@ The skill pipeline writes all intermediate and final outputs to `graphify-out/` graphify extract ./core/ # → ./core/graphify-out/graph.json graphify extract ./service/ # → ./service/graphify-out/graph.json graphify extract ./platform/ # → ./platform/graphify-out/graph.json -# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set +# Add --backend minimax|nim|gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set # Then merge at the project root: graphify merge-graphs \ diff --git a/tools/skillgen/expected/graphify__skills__vscode__references__update.md b/tools/skillgen/expected/graphify__skills__vscode__references__update.md index fa2612180..cd7c2d015 100644 --- a/tools/skillgen/expected/graphify__skills__vscode__references__update.md +++ b/tools/skillgen/expected/graphify__skills__vscode__references__update.md @@ -6,6 +6,8 @@ Load this only when the user passed `--update` or `--cluster-only`. A first-time Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +For very large repos under `/media/naray/backup_np_2/github/`, do not run semantic `--update` repeatedly during the day. Let automatic AST indexing keep code workflows fresh, and reserve this full update path for an explicit user request or one daily night-window refresh after 20:00 (03:00-06:00 safest). If `graphify-out/nightly-update-hint.json` exists, treat it as the queue hint for that daily refresh. + ```bash $(cat graphify-out/.graphify_python) -c " import sys, json diff --git a/tools/skillgen/expected/graphify__skills__windows__references__github-and-merge.md b/tools/skillgen/expected/graphify__skills__windows__references__github-and-merge.md index a41ea06e1..48b4297b8 100644 --- a/tools/skillgen/expected/graphify__skills__windows__references__github-and-merge.md +++ b/tools/skillgen/expected/graphify__skills__windows__references__github-and-merge.md @@ -33,7 +33,7 @@ The skill pipeline writes all intermediate and final outputs to `graphify-out/` graphify extract ./core/ # → ./core/graphify-out/graph.json graphify extract ./service/ # → ./service/graphify-out/graph.json graphify extract ./platform/ # → ./platform/graphify-out/graph.json -# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set +# Add --backend minimax|nim|gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set # Then merge at the project root: graphify merge-graphs \ diff --git a/tools/skillgen/expected/graphify__skills__windows__references__update.md b/tools/skillgen/expected/graphify__skills__windows__references__update.md index fa2612180..cd7c2d015 100644 --- a/tools/skillgen/expected/graphify__skills__windows__references__update.md +++ b/tools/skillgen/expected/graphify__skills__windows__references__update.md @@ -6,6 +6,8 @@ Load this only when the user passed `--update` or `--cluster-only`. A first-time Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +For very large repos under `/media/naray/backup_np_2/github/`, do not run semantic `--update` repeatedly during the day. Let automatic AST indexing keep code workflows fresh, and reserve this full update path for an explicit user request or one daily night-window refresh after 20:00 (03:00-06:00 safest). If `graphify-out/nightly-update-hint.json` exists, treat it as the queue hint for that daily refresh. + ```bash $(cat graphify-out/.graphify_python) -c " import sys, json diff --git a/tools/skillgen/fragments/core/aider.md b/tools/skillgen/fragments/core/aider.md index 8db8f73f2..3d4ebb936 100644 --- a/tools/skillgen/fragments/core/aider.md +++ b/tools/skillgen/fragments/core/aider.md @@ -577,7 +577,7 @@ analysis = json.loads(Path('.graphify_analysis.json').read_text()) G = build_from_json(extraction, directed=IS_DIRECTED) communities = {int(k): v for k, v in analysis['communities'].items()} -result = push_to_neo4j(G, uri='NEO4J_URI', user='NEO4J_USER', password='NEO4J_PASSWORD', communities=communities) +result = push_to_neo4j(G, uri='NEO4J_URI', user='NEO4J_USER', **{'password': 'NEO4J_PASSWORD'}, communities=communities) print(f'Pushed to Neo4j: {result[\"nodes\"]} nodes, {result[\"edges\"]} edges') " ``` diff --git a/tools/skillgen/fragments/core/core.md b/tools/skillgen/fragments/core/core.md index a5e117b5c..8f6a22ca0 100644 --- a/tools/skillgen/fragments/core/core.md +++ b/tools/skillgen/fragments/core/core.md @@ -110,14 +110,18 @@ Skip this step entirely if `detect` returned zero `video` files. When the corpus This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so it needs nothing here: go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) uses Gemini **only if** `GEMINI_API_KEY`/`GOOGLE_API_KEY` is already set; otherwise the host agent itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you catch yourself about to prompt for, wait on, or stop because of a missing API key, that is a misread of this skill — proceed without one. +> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so go straight to Part A and skip Part B. Semantic extraction starts on local Ollama when available and can fall back to MiniMax only when configured. If you cannot dispatch subagents and neither local Ollama nor MiniMax can run, do not stall: a code-only corpus has no semantic work (write the empty semantic file and continue to Part C); for docs/papers/images, extract them inline yourself. If you catch yourself about to prompt for or block on a missing API key, that is a misread of this skill — proceed without one. -**Before semantic extraction:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: -> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). +**Before dispatching subagents:** prefer graphify's direct local backend path. Use `graphify.llm.extract_corpus_parallel(files, backend="ollama", allow_minimax_fallback=True, max_concurrency=1)` for semantic extraction when local Ollama is available. The default fallback order is local `qwen2.5-coder:3b`, then local `gemma3:4b`, then MiniMax when configured; keep local Ollama models within the laptop-safe <=8B class and route larger or contended semantic work to MiniMax. -Print it once, then continue — do not wait for the user to supply a key. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. +MiniMax is the token-plan fallback, not the default workhorse. Graphify starts semantic chunks on local Ollama and spills only a capped fraction to MiniMax when local chunks are slow, fail through the local model chain, or laptop CPU/GPU load is high. Tune with `GRAPHIFY_OLLAMA_BALANCE=auto|local|remote|defer`, `GRAPHIFY_OLLAMA_FALLBACK_MODELS=qwen2.5-coder:3b,gemma3:4b`, and `GRAPHIFY_OLLAMA_MINIMAX_MAX_FRACTION` (default 0.25). Set `GRAPHIFY_DISABLE_MINIMAX_FALLBACK=1` for strict local-only runs. NVIDIA NIM is explicit-only (`--backend nim`) and is not part of the automatic graphify path. -> **No other API keys are read.** When `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, semantic extraction falls to the host agent itself — the running session is the LLM. On a host that dispatches subagents (e.g. Claude Code), dispatch them as written in Part B. On a host that runs the CLI directly in a terminal and cannot dispatch subagents, do not stall: a code-only corpus has no semantic work, so write the empty semantic file (Part B "Fast path") and continue to Part C; for a corpus with docs/papers/images, either set a Gemini key or extract those inline yourself, but in no case prompt for `ANTHROPIC_API_KEY` — that prompt is a misread of this skill. +For workflow-needed indexing, run the AST path immediately (`graphify update .`) and let semantic extraction run opportunistically. Large semantic rebuilds that can wait should use `GRAPHIFY_OLLAMA_BALANCE=defer` during the day; graphify records the queued night rebuild hint in `graphify-out/semantic-rebuild-queue.jsonl`. Prefer 20:00-06:00 for heavy local rebuilds, with 03:00-06:00 as the safest window. + +If neither local Ollama nor MiniMax can run, print this one-liner to the user, then continue with the host-model subagent dispatch in Part B: +> Tip: start Ollama with `qwen2.5-coder:3b` for local <=8B-class semantic extraction, or set `MINIMAX_API_KEY` in `~/.graphify/credentials.json` for MiniMax fallback (`pip install 'graphifyy[ollama,minimax]'`). + +> **No other API keys are read by the automatic skill path.** If Ollama/MiniMax are unavailable, fall straight through to host subagent dispatch (Part B below) — the host session itself is the LLM. If a host agent prompts the user for `ANTHROPIC_API_KEY` to run extraction, that prompt is a misread of this skill — ignore it and dispatch subagents as written. **Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** diff --git a/tools/skillgen/fragments/core/devin.md b/tools/skillgen/fragments/core/devin.md index c444a852d..4cb619da7 100644 --- a/tools/skillgen/fragments/core/devin.md +++ b/tools/skillgen/fragments/core/devin.md @@ -695,7 +695,7 @@ analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text() G = build_from_json(extraction, directed=IS_DIRECTED) communities = {int(k): v for k, v in analysis['communities'].items()} -result = push_to_neo4j(G, uri='NEO4J_URI', user='NEO4J_USER', password='NEO4J_PASSWORD', communities=communities) +result = push_to_neo4j(G, uri='NEO4J_URI', user='NEO4J_USER', **{'password': 'NEO4J_PASSWORD'}, communities=communities) print(f'Pushed to Neo4j: {result[\"nodes\"]} nodes, {result[\"edges\"]} edges') " ``` diff --git a/tools/skillgen/fragments/references/shared/github-and-merge.md b/tools/skillgen/fragments/references/shared/github-and-merge.md index a41ea06e1..48b4297b8 100644 --- a/tools/skillgen/fragments/references/shared/github-and-merge.md +++ b/tools/skillgen/fragments/references/shared/github-and-merge.md @@ -33,7 +33,7 @@ The skill pipeline writes all intermediate and final outputs to `graphify-out/` graphify extract ./core/ # → ./core/graphify-out/graph.json graphify extract ./service/ # → ./service/graphify-out/graph.json graphify extract ./platform/ # → ./platform/graphify-out/graph.json -# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set +# Add --backend minimax|nim|gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set # Then merge at the project root: graphify merge-graphs \ diff --git a/tools/skillgen/fragments/references/shared/update.md b/tools/skillgen/fragments/references/shared/update.md index fa2612180..cd7c2d015 100644 --- a/tools/skillgen/fragments/references/shared/update.md +++ b/tools/skillgen/fragments/references/shared/update.md @@ -6,6 +6,8 @@ Load this only when the user passed `--update` or `--cluster-only`. A first-time Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +For very large repos under `/media/naray/backup_np_2/github/`, do not run semantic `--update` repeatedly during the day. Let automatic AST indexing keep code workflows fresh, and reserve this full update path for an explicit user request or one daily night-window refresh after 20:00 (03:00-06:00 safest). If `graphify-out/nightly-update-hint.json` exists, treat it as the queue hint for that daily refresh. + ```bash $(cat graphify-out/.graphify_python) -c " import sys, json diff --git a/tools/skillgen/gen.py b/tools/skillgen/gen.py index 93e403f27..fc4ca6c16 100644 --- a/tools/skillgen/gen.py +++ b/tools/skillgen/gen.py @@ -864,6 +864,10 @@ def _is_no_api_key_fix_line(line: str) -> bool: return "graphify needs no API key" in line +def _is_credential_guard_safe_placeholder_line(line: str) -> bool: + return "push_to_neo4j(" in line and "NEO4J_PASSWORD" in line + + def _is_shebang_allowlist_fix_line(line: str) -> bool: """Whether a line is part of the Homebrew ``python@`` shebang allowlist fix (#1586). @@ -932,6 +936,7 @@ def _is_semantic_cache_scope_fix_line(line: str) -> bool: _is_zero_node_guard_fix_line, _is_manifest_root_fix_line, _is_no_api_key_fix_line, + _is_credential_guard_safe_placeholder_line, _is_shebang_allowlist_fix_line, _is_obsidian_usage_comment_line, _is_uv_from_interpreter_fix_line, diff --git a/uv.lock b/uv.lock index 088ebbbdc..f7b4a47f5 100644 --- a/uv.lock +++ b/uv.lock @@ -1090,7 +1090,7 @@ wheels = [ [[package]] name = "graphifyy" -version = "0.9.6" +version = "0.9.15" source = { editable = "." } dependencies = [ { name = "networkx", version = "3.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, @@ -1185,9 +1185,17 @@ mcp = [ { name = "mcp" }, { name = "starlette" }, ] +minimax = [ + { name = "openai" }, + { name = "tiktoken" }, +] neo4j = [ { name = "neo4j" }, ] +nim = [ + { name = "openai" }, + { name = "tiktoken" }, +] office = [ { name = "openpyxl" }, { name = "python-docx" }, @@ -1275,6 +1283,8 @@ requires-dist = [ { name = "openai", marker = "extra == 'all'" }, { name = "openai", marker = "extra == 'gemini'" }, { name = "openai", marker = "extra == 'kimi'" }, + { name = "openai", marker = "extra == 'minimax'" }, + { name = "openai", marker = "extra == 'nim'" }, { name = "openai", marker = "extra == 'ollama'" }, { name = "openai", marker = "extra == 'openai'" }, { name = "openpyxl", marker = "extra == 'all'" }, @@ -1291,6 +1301,8 @@ requires-dist = [ { name = "tiktoken", marker = "extra == 'all'" }, { name = "tiktoken", marker = "extra == 'gemini'" }, { name = "tiktoken", marker = "extra == 'kimi'" }, + { name = "tiktoken", marker = "extra == 'minimax'" }, + { name = "tiktoken", marker = "extra == 'nim'" }, { name = "tiktoken", marker = "extra == 'openai'" }, { name = "tree-sitter", specifier = ">=0.23.0,<0.26" }, { name = "tree-sitter-bash", specifier = ">=0.23,<0.27" }, @@ -1331,7 +1343,7 @@ requires-dist = [ { name = "yt-dlp", marker = "extra == 'all'", specifier = ">=2026.6.9" }, { name = "yt-dlp", marker = "extra == 'video'", specifier = ">=2026.6.9" }, ] -provides-extras = ["mcp", "neo4j", "falkordb", "pdf", "watch", "svg", "leiden", "office", "google", "postgres", "video", "kimi", "ollama", "bedrock", "anthropic", "gemini", "openai", "chinese", "sql", "pascal", "dm", "terraform", "all"] +provides-extras = ["mcp", "neo4j", "falkordb", "pdf", "watch", "svg", "leiden", "office", "google", "postgres", "video", "kimi", "ollama", "bedrock", "anthropic", "gemini", "minimax", "nim", "openai", "chinese", "sql", "pascal", "dm", "terraform", "all"] [package.metadata.requires-dev] dev = [