From e23508a5bbfd54ee681835483c360a53e2186d8d Mon Sep 17 00:00:00 2001 From: Swapnil <78632212+swapnilpaliwal-sd@users.noreply.github.com> Date: Mon, 28 Sep 2026 01:13:46 -0700 Subject: [PATCH] index: publish each language's graph as soon as it is solved (#1555) A repository in several languages was solved one language after another, and nothing was published until the last one finished. The main language's graph sat finished while another language's rules compiled, and every query in between waited on the build lock. - bin/axiomcode `all --progress ` appends " ok " or " failed" as each language finishes; without --language the largest language is solved first. - axiomcode-build runs the engine in the background (own process group, stopped with the build), solves the main language first and the rest by file count, publishes the main graph (index, stamp, file table, baseline, pointer) the moment it is solved, and moves every other graph into .axiomcode/lang as it arrives. .axiomcode/out/building lists the languages still to come. An engine without --progress is published at the end as before. - ax_fresh: status and query read the pending list; an answer from a current main graph is given at once and carries one line naming the languages it cannot see yet, or whose code comes from their previous graph. build-seconds records the time to the main graph. - ax_contract.ensure_graph: a first query waits for graph.sqlite, not for the whole build; the build's output goes to a log copied to stderr, so stdout carries only the answer. - tests/publish_order.py: C# main with Python and Java beside it, the others held back by a wrapped engine; one-language control. --- bin/axiomcode | 14 +- .../skills/axiomcode/scripts/ax_contract.py | 49 +++- .../skills/axiomcode/scripts/ax_fresh.py | 61 ++++- .../skills/axiomcode/scripts/axiomcode-build | 192 +++++++++----- tests/README.md | 4 + tests/publish_order.py | 234 ++++++++++++++++++ 6 files changed, 472 insertions(+), 82 deletions(-) create mode 100644 tests/publish_order.py diff --git a/bin/axiomcode b/bin/axiomcode index fa44d4bbb..3e1d32c87 100755 --- a/bin/axiomcode +++ b/bin/axiomcode @@ -20,6 +20,8 @@ # --language [,…] Restrict the run to these languages: java, typescript, python, # javascript or csharp. # --debug Also write csv/*.csv and keep raw/ beside each graph. +# --progress Append " ok " or " failed" to as each language +# finishes, so a caller can use one graph while the next is solved. # # Engine options: # --dispatch-cap --lib-depth --jdk-depth --taint @@ -188,10 +190,10 @@ case "$cmd" in ;; all) need_parser - lang=""; src=""; out=""; version=""; libs=""; popts=(); rest=(); pos=() + lang=""; src=""; out=""; version=""; libs=""; progress=""; popts=(); rest=(); pos=() while [ $# -gt 0 ]; do case "$1" in --language) lang="$2"; shift 2;; --src) src="$2"; shift 2;; --out) out="$2"; shift 2;; - --library) libs="$2"; shift 2;; + --library) libs="$2"; shift 2;; --progress) progress="$2"; shift 2;; --version|--slug) version="$2"; shift 2;; --exclude-tests) popts+=(--exclude-tests); shift;; --intermediate|--dispatch-cap|--lib-depth|--jdk-depth|--taint) rest+=("$1" "$2"); shift 2;; --debug) rest+=(--debug); shift;; @@ -259,7 +261,10 @@ case "$cmd" in # which languages the IR holds — one folder each — and which of them have an engine found=(); for d in "$ir"/*/; do l="$(basename "$d")"; [ -d "$ROOT/graph/$l/engine" ] && found+=("$l"); done [ ${#found[@]} -gt 0 ] || { echo "❌ no language with an engine in the IR (folders: $(ls "$ir" | tr '\n' ' '))" >&2; exit 1; } - langs=("${found[@]}") + # THE LARGEST LANGUAGE FIRST (#1555): a caller that publishes each graph as it is solved (--progress) gets the one + # most of the repository is written in first, not whichever sorts first. The IR's size stands in for its share; a + # caller that names --language chooses the order itself. + langs=($(for l in "${found[@]}"; do echo "$(du -sk "$ir/$l" 2>/dev/null | cut -f1) $l"; done | sort -s -k1,1nr | cut -d' ' -f2)) if [ ${#want[@]} -eq 1 ]; then case " ${found[*]} " in *" $lang "*) langs=("$lang");; *) echo "❌ --language=$lang, but the IR holds only: ${found[*]}" >&2; exit 1;; esac elif [ ${#want[@]} -gt 1 ]; then @@ -287,9 +292,10 @@ case "$cmd" in echo "▶ solving $l → $out/$l/graph.sqlite" if "$0" engine --language "$l" --client-ir "$ir/$l" --out "$out/$l" --intermediate "$int" --library "$libroots" \ --meta "source_version=$version" --meta "source_dir=$(cd "$src" && pwd)" ${rest[@]+"${rest[@]}"}; then - solved+=("$l") + solved+=("$l"); [ -z "$progress" ] || echo "$l ok $out/$l/graph.sqlite" >> "$progress" else failed+=("$l"); echo "❌ $l failed; continuing with the remaining languages" >&2 + [ -z "$progress" ] || echo "$l failed" >> "$progress" fi done for l in ${solved[@]+"${solved[@]}"}; do echo "$out/$l/graph.sqlite"; done diff --git a/plugins/axiomcode/skills/axiomcode/scripts/ax_contract.py b/plugins/axiomcode/skills/axiomcode/scripts/ax_contract.py index 5da78a88b..252f79a8a 100644 --- a/plugins/axiomcode/skills/axiomcode/scripts/ax_contract.py +++ b/plugins/axiomcode/skills/axiomcode/scripts/ax_contract.py @@ -391,9 +391,9 @@ def ensure_graph(repo, db): # a build is running: never a silent wait. Under the MCP server answer at once with the stage it is at; from a # shell wait for it, printing the stage as it moves; either way, use the graph the moment it exists if auto and not os.environ.get('AXIOMCODE_BUILD_NOWAIT'): - print(f"a graph build is already running for {repo}; waiting for it …", file=sys.stderr) - _follow(rr, lambda: ax_fresh.building(rr)) - if os.path.exists(db): return True + print(f"a graph build is already running for {repo}; waiting for its first graph …", file=sys.stderr) + _follow(rr, lambda: ax_fresh.building(rr) and not os.path.exists(db)) + if os.path.exists(db): return _published(rr) if auto and os.environ.get('AXIOMCODE_BUILD_NOWAIT'): _BUILD_NOTE.append(building_note(rr)); return False if not auto or os.environ.get('AXIOMCODE_GRAPH'): return False @@ -411,9 +411,9 @@ def ensure_graph(repo, db): nowait = bool(os.environ.get('AXIOMCODE_BUILD_NOWAIT')) if ax_fresh.building(repo): if nowait: _BUILD_NOTE.append(building_note(repo)); return False - print(f"a graph build is already running for {repo}; waiting for it …", file=sys.stderr) - _follow(repo, lambda: ax_fresh.building(repo)) - return os.path.exists(db) + print(f"a graph build is already running for {repo}; waiting for its first graph …", file=sys.stderr) + _follow(repo, lambda: ax_fresh.building(repo) and not os.path.exists(db)) + return os.path.exists(db) and _published(repo) env = None if ax_fresh.has_graph(rr): # a graph WAS built here and its pointer is broken: a repair, which keeps the baseline `changed` and test-impact @@ -432,14 +432,43 @@ def ensure_graph(repo, db): up = time.time() + 10 while p.poll() is None and not ax_fresh.building(repo) and time.time() < up: time.sleep(0.05) end = time.time() + float(os.environ.get('AXIOMCODE_BUILD_WAIT') or 60) - while p.poll() is None and time.time() < end: time.sleep(0.5) - if p.poll() is None: _BUILD_NOTE.append(building_note(repo)); return False + while p.poll() is None and not os.path.exists(db) and time.time() < end: time.sleep(0.5) + if p.poll() is None: return _published(repo) if os.path.exists(db) else (_BUILD_NOTE.append(building_note(repo)) or False) return p.returncode == 0 and os.path.exists(db) - p = subprocess.Popen([bash, build, repo], env=env) - _follow(repo, lambda: p.poll() is None) + # THE ANSWER WAITS FOR ITS GRAPH, NOT FOR THE WHOLE BUILD (#1555). The build publishes the main language's graph as + # soon as that language is solved and goes on solving the others under its lock, so the wait ends when graph.sqlite + # appears. The build writes to a log, copied here to stderr as it comes, not to this process's pipes: a caller that + # captures them (an agent's shell tool) reads until they close, and a build still solving other languages would hold + # them open to its end. + os.makedirs(os.path.join(repo, '.axiomcode'), exist_ok=True) + logp = os.path.join(repo, '.axiomcode', 'first-build.log') + with open(logp, 'w') as log: + p = subprocess.Popen([bash, build, repo], stdin=subprocess.DEVNULL, stdout=log, stderr=log, close_fds=True, env=env) + shown = [0] + def relay(): + try: + with open(logp, 'rb') as f: f.seek(shown[0]); b = f.read() + except OSError: return + shown[0] += len(b); sys.stderr.write(b.decode('utf-8', 'replace')); sys.stderr.flush() + _follow(repo, lambda: relay() or (p.poll() is None and not os.path.exists(db))) + relay() + if p.poll() is None: return os.path.exists(db) and _published(repo) return p.returncode == 0 and os.path.exists(db) +def _published(repo): + """the main graph is out while the build goes on solving the repository's other languages: the answer is given now, + and names on stderr (a `graph refresh:` line, which the MCP server carries into the answer) what it cannot see yet""" + import ax_fresh + n = '' + for _ in range(20): # the build names what is still to come just before the pointer moves + n = ax_fresh.note(ax_fresh.status(repo)) + if n or not ax_fresh.building(repo): break + time.sleep(0.1) + if n: print(n, file=sys.stderr) + return True + + _BUILD_NOTE = [] TICK = 20.0 diff --git a/plugins/axiomcode/skills/axiomcode/scripts/ax_fresh.py b/plugins/axiomcode/skills/axiomcode/scripts/ax_fresh.py index 8b375db2f..e94735c23 100644 --- a/plugins/axiomcode/skills/axiomcode/scripts/ax_fresh.py +++ b/plugins/axiomcode/skills/axiomcode/scripts/ax_fresh.py @@ -325,6 +325,18 @@ def wait_build(repo, seconds, say=True): time.sleep(0.5) return True +def pending(repo): + """what a running build has still to publish once the main graph is out (axiomcode-build, #1555): {} when nothing, + or when no build holds the lock (a build that was killed leaves the file, and it means nothing then). Else + `pending` (the languages, in solve order), `pending_old` (those a previous graph still answers for), and `first` + (a first build: there is no earlier graph of anything to wait for).""" + try: rows = [l.split() for l in open(os.path.join(out_dir(repo), 'building')) if l.strip()] + except OSError: return {} + if not rows or not building(repo): return {} + langs = [r[0] for r in rows] + old = [l for l in langs if os.path.isfile(os.path.join(repo, '.axiomcode', 'lang', l, 'out', 'graph.sqlite'))] + return dict(pending=langs, pending_old=old, first=all(len(r) > 1 and r[1] == 'first' for r in rows)) + def status(repo): if not has_graph(repo): return dict(state='no graph') if graph_broken(repo): return dict(state='building' if building(repo) else 'stale', changed=['.axiomcode/out/graph.sqlite (the pointer to the graph is broken)'], added=[], removed=[]) @@ -333,7 +345,10 @@ def status(repo): if c is None: return dict(state='unknown', note='the graph predates the file table; the next `axiomcode index` records it') changed, added, removed = c busy = building(repo) - if not (changed or added or removed): return dict(state='building' if busy else 'fresh') + if not (changed or added or removed): + # a build that already published this tree's main graph and is still solving other languages: the graph a query + # reads is current, and waiting would be waiting for other languages' compiles + return dict(state='building', **pending(repo)) if busy else dict(state='fresh') d = dict(state='building' if busy else 'stale', changed=changed, added=added, removed=removed) if not busy and st.get('failed_table') == change_key(c): d['failed'] = st.get('failed_log', ''); d['failed_reason'] = st.get('failed_reason', '') return d @@ -438,12 +453,16 @@ def failure_reason(out): def wait(repo, seconds): """kick, then wait until the graph is fresh or `seconds` have passed. Returns the status at the end.""" - if not enabled(repo): return dict(state='off') # switched off, or a graph placed by AXIOMCODE_GRAPH: nothing to say + if not enabled(repo): # switched off, or a graph placed by AXIOMCODE_GRAPH: nothing to say, + p = {} if os.environ.get('AXIOMCODE_GRAPH') or not has_graph(repo) else pending(repo) # but the languages a build + return dict(state='building', **p) if p else dict(state='off') # has yet to publish end = time.time() + seconds while True: s = status(repo) if s['state'] == 'unknown': kick(repo, 'a query'); return s # a graph from before the file table: its first refresh records one - if s['state'] in ('fresh', 'no graph') or s.get('failed'): return s + # a FIRST build's other languages are not waited for: there is no graph of theirs to refresh, only a compile + # that can take an hour (#1555). A rebuild's are, for the bounded while any refresh is. + if s['state'] in ('fresh', 'no graph') or s.get('failed') or s.get('first'): return s if s['state'] == 'stale': kick(repo, 'a query') # idempotent: a no-op while a worker holds its lock if time.time() >= end: return s time.sleep(0.5) @@ -484,21 +503,34 @@ def edited(s): """the files a stale graph predates: changed, added and removed since it was built""" return list(s.get('changed', [])) + list(s.get('added', [])) + list(s.get('removed', [])) +def pending_note(s): + """one line naming the languages a build has published the main graph without, or '' (#1555)""" + if not s.get('pending'): return '' + new = [l for l in s['pending'] if l not in s.get('pending_old', [])]; old = s.get('pending_old', []) + says = [] + if new: says.append(f"this answer has no {', '.join(new)} code yet") + if old: says.append(f"its {', '.join(old)} code comes from the previous graph") + return (f"graph refresh: the {', '.join(s['pending'])} graph{'s are' if len(s['pending']) > 1 else ' is'} still being built; " + + '; '.join(says) + "; ask again when `axiomcode index` finishes") + def note(s, marked=None): """one line for an answer given from a graph that is behind the files, or '' when it is not. `marked` is how many of the answer's rows lie in those files (and carry the mark), None when the answer was not looked at""" - if s.get('state') not in ('stale', 'building'): return '' + # the languages a running build has still to publish (#1555) get a line of their own, before any line about edits + first = pending_note(s) + if s.get('state') not in ('stale', 'building'): return first files = edited(s) - if not files: return '' + if not files: return first + first = first + '\n' if first else '' head = ', '.join(files[:5]) + (f" … +{len(files) - 5}" if len(files) > 5 else '') rows = ('' if marked is None else f"; {marked} row(s) lie in those files and are marked {MARK.strip()}: read them for their current text" if marked else "; no row of this answer lies in those files") if s.get('failed'): why = s.get('failed_reason') or '' - return (f"graph refresh: the last rebuild FAILED" + (f" — {why}" if why else '') + f" (see {s['failed']}); this answer is from the " + return first + (f"graph refresh: the last rebuild FAILED" + (f" — {why}" if why else '') + f" (see {s['failed']}); this answer is from the " f"previous graph, which predates edits to {head}" + rows) - return (f"graph refresh: {'rebuilding' if s['state'] == 'building' else 'queued'} — this answer is from the previous graph, " + return first + (f"graph refresh: {'rebuilding' if s['state'] == 'building' else 'queued'} — this answer is from the previous graph, " f"which predates edits to {head}" + (rows if marked is not None else '; read those files for their current text') + "; --fresh waits for the rebuild") @@ -647,10 +679,21 @@ def query(repo, verb, argv, fresh=False): def run(): return subprocess.run(argv, stdout=subprocess.PIPE) def passthrough(): os.execvp(argv[0], argv) - if not enabled(repo): passthrough() + def with_note(n): # the answer as it is, then one line about it on stderr + r = subprocess.run(argv); print(n, file=sys.stderr); return r.returncode + if not enabled(repo): + # the refresh switched off still leaves a build that is solving the repository's other languages (#1555); a + # graph placed by AXIOMCODE_GRAPH is not this build's, and says nothing + n = '' if os.environ.get('AXIOMCODE_GRAPH') or not has_graph(repo) else pending_note(pending(repo)) + if not n: passthrough() + return with_note(n) s = status(repo) if s['state'] == 'unknown': kick(repo, 'a query'); passthrough() - if s['state'] in ('fresh', 'no graph') or not edited(s): passthrough() + if s['state'] in ('fresh', 'no graph') or not edited(s): + # a build that published this tree's main graph and is solving the others (#1555): the answer is current and is + # given now, and says which languages it cannot see yet + if not pending_note(s): passthrough() + return with_note(pending_note(s)) if not s.get('failed'): kick(repo, 'a query') as_json = '--json' in argv if fresh and not s.get('failed'): diff --git a/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-build b/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-build index feeeb48af..a2c2b97d2 100755 --- a/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-build +++ b/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-build @@ -11,6 +11,8 @@ # A repository in several languages gets one graph per language. The main language's is the one it always had, # .axiomcode/out/graph.sqlite; every other language's is .axiomcode/lang//out/graph.sqlite, and the query verbs # (the `axiomcode` dispatcher) ask each of them. Each graph is one language: no call is followed from one to another. +# Each is published as soon as its own language is solved, the main one first, so a query never waits for another +# language's compile; until the last one is in, .axiomcode/out/building names the languages still to come. set -euo pipefail H="$(cd "$(dirname "$0")" && pwd)" REPO="$(cd "${1:-.}" && pwd)"; SRC="$REPO/${AXIOMCODE_SRC:-}"; SRC="${SRC%/}" @@ -98,13 +100,21 @@ else fi unset AXIOMCODE_GRAPH_LANG # a build writes every language's graph, never one chosen for a query OTHER_LANGS=""; [ "$LANGS" != "$LANG_ARG" ] && OTHER_LANGS="${LANGS#*,}" +# THE ORDER THE ENGINE SOLVES IN (#1555): the main language first, then the rest by file count, most first. Each graph is +# published the moment its language is solved, so this is the order they become queryable in. LANGS keeps its own order: +# it is part of the stamp and the file table, and a new order there would rebuild every mixed repository once. +SOLVE="$LANG_ARG" +if [ -n "$OTHER_LANGS" ]; then + files_of(){ case "$1" in (java) echo "$ja";; (typescript) echo "$ts";; (python) echo "$py";; (javascript) echo "$js";; (csharp) echo "$cs";; (*) echo 0;; esac; } + for l in $(for o in ${OTHER_LANGS//,/ }; do echo "$(files_of "$o") $o"; done | sort -s -k1,1nr | cut -d' ' -f2); do SOLVE="$SOLVE,$l"; done +fi OUT="$REPO/.axiomcode/out"; mkdir -p "$OUT" # ONE BUILD AT A TIME PER REPOSITORY. The background refresher (ax_fresh.py) and an explicit `axiomcode index` both # write $OUT/$LANG_ARG; two at once clobber each other's graph. The lock is on fd 9, which this shell holds for its # whole life, so the OS drops it when the build ends however it ends: no stale lock to time out. exec 9>"$REPO/.axiomcode/build.lock" if ! python3 "$H/ax_fresh.py" lock 9 --try; then - [ -n "${AXIOMCODE_BACKGROUND:-}" ] || echo "a graph build is already running for $REPO (the background refresh); waiting for it …" + [ -n "${AXIOMCODE_BACKGROUND:-}" ] || echo "a graph build is already running for $REPO; waiting for it …" python3 "$H/ax_fresh.py" lock 9 || { echo "could not take the build lock ($REPO/.axiomcode/build.lock)"; exit 75; } fi BUILD_T0=$SECONDS # how long a build takes to a usable graph: a query decides from it whether to wait (#1595) @@ -249,6 +259,9 @@ fi # build succeeds every verb keeps answering from it. A graph built for another language is not in the # way and is left where it is — which is also what $OUT/graph.sqlite still points at. PREV="$OUT/.previous-$LANG_ARG" +# A FIRST BUILD has no graph for any query to wait for; a rebuild (a refresh, an upgrade that adds a language) has the +# previous one, and a query waits a bounded while for its successor as it always did. $PENDING says which it is. +MODE=refresh; [ -e "$OUT/graph.sqlite" ] || MODE=first rm -rf "$PREV" # graph.sqlite NEVER DANGLES (#1305). With a refresher running after every edit, queries arrive mid-build all the time. # Moving the directory first and re-pointing after left a moment with no graph.sqlite at all, and a verb that finds @@ -287,27 +300,11 @@ restore(){ rm -rf "$OUT/$LANG_ARG"; if [ -d "$PREV" ]; then mv "$PREV" "$OUT/$LA [ -f "$OUT/.live.sqlite" ] && [ "$(real "$OUT/graph.sqlite")" != "$(real "$OUT/.live.sqlite")" ] && rm -f "$OUT/.live.sqlite"; return 0; } LIB=(); [ -n "${AXIOMCODE_LIBRARY:-}" ] && LIB=(--library "$AXIOMCODE_LIBRARY") ENGINE_RC=0 -"$ENGINE/bin/axiomcode" all "$SRC" "$OUT" --language "$LANGS" ${LIB[@]+"${LIB[@]}"} > "$REPO/.axiomcode/build.log" 2>&1 || ENGINE_RC=$? -# one language failing does not cost the others their graphs (the engine solves each on its own): only the MAIN -# language's failure fails the build. Another language's is named below, never dropped in silence. -if [ "$ENGINE_RC" != 0 ] && { [ -z "$OTHER_LANGS" ] || [ ! -f "$OUT/$LANG_ARG/graph.sqlite" ]; }; then - restore; echo "build failed — the graph that was there is untouched; see $REPO/.axiomcode/build.log"; tail -5 "$REPO/.axiomcode/build.log"; exit 1 -fi -# the new database: a FILE under $OUT/$LANG_ARG. Matched as `*/$LANG_ARG/*` it was any path with the language's name -# in it, the repository's own included: a repository at .../python/app matched the graph.sqlite SYMLINK, the link was -# pointed at itself, the previous graph deleted, and every verb after it reported no graph. -DB="$(find "$OUT/$LANG_ARG" -type f -name graph.sqlite 2>/dev/null | head -1)"; NOT_OTHER=(); for l in ${OTHER_LANGS//,/ }; do NOT_OTHER+=(-not -path "$OUT/$l/*"); done # never another language's graph -[ -n "$DB" ] || DB="$(find "$OUT" -type f -name graph.sqlite -not -path "$PREV/*" ${NOT_OTHER[@]+"${NOT_OTHER[@]}"} | head -1)" -[ -n "$DB" ] || { restore; echo "build produced no graph.sqlite — the graph that was there is untouched; see $REPO/.axiomcode/build.log"; exit 1; } -# A QUERY NEVER READS A HALF-BUILT GRAPH. The new database gets its symbols, refs and views BEFORE graph.sqlite points -# at it; until the pointer moves, every verb (and the hooks) keeps answering from the previous graph. Only then is the -# old one removed: a reader that opened it keeps the file it opened. -AXIOMCODE_INDEX_DB="$DB" python3 "$H/axiomcode-index" "$REPO" || { restore; echo "indexing the new graph failed — the graph that was there is untouched"; exit 1; } +REASON="$(printf %s "${AXIOMCODE_REFRESH_REASON:-axiomcode index}" | tr -d "'")" # WHEN AND WHY THIS GRAPH WAS BUILT, in the graph itself (index_meta): refreshed_at (UTC), refresh_reason (`axiomcode # index`, or what the background refresher saw: files changed, HEAD moved, found by an edit, a query or the timer), and # the tree and commit it describes. Written before the swap: a write to the live graph later would change its mtime, # which keys the impact-facts cache, and cost the next query a re-export. -REASON="$(printf %s "${AXIOMCODE_REFRESH_REASON:-axiomcode index}" | tr -d "'")" stamp_meta(){ python3 - "$1" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$REASON" "$INDEXED_TREE" "$HEAD_SHA" <<'PY' 2>/dev/null || true import sqlite3, sys db, at, reason, tree, commit = sys.argv[1:6] @@ -317,54 +314,131 @@ with sqlite3.connect(db) as c: ('refreshed_tree', tree), ('refreshed_commit', commit)]) PY } -stamp_meta "$DB" -# EVERY OTHER LANGUAGE'S GRAPH, indexed where the engine wrote it and only then moved into place, as the main one is. -# One that did not build is NAMED: its previous graph, if any, keeps answering, and this build says it is behind. -READY=(); FAILED=() -for l in ${OTHER_LANGS//,/ }; do - LDB="$OUT/$l/graph.sqlite" - if [ -f "$LDB" ] && AXIOMCODE_INDEX_DB="$LDB" python3 "$H/axiomcode-index" "$REPO"; then stamp_meta "$LDB"; READY+=("$l"); else FAILED+=("$l"); fi -done -echo "$STAMP" > "$OUT/stamp" -# the engine that built this graph: the next build, the background refresh above all, starts its search there (see the top) -printf '%s\n' "$ENGINE" > "$REPO/.axiomcode/engine.new" && mv -f "$REPO/.axiomcode/engine.new" "$REPO/.axiomcode/engine" || true -if [ -n "$INDEXED_TREE" ]; then echo "$INDEXED_TREE" > "$OUT/indexed-tree"; else rm -f "$OUT/indexed-tree"; fi -set_base "$INDEXED_TREE" -point "$DB" || { restore; exit 1; }; rm -f "$OUT/.live.sqlite" -if [ -f "$OUT/.files.json.new" ]; then mv "$OUT/.files.json.new" "$OUT/files.json"; else rm -f "$OUT/files.json"; fi -# the graph now matches the files: record how long that took, and whether it included compiling the engine's rules (a -# one-time cost a refresh does not pay, so such a duration predicts nothing) -printf '%s %s\n' "$((SECONDS - BUILD_T0))" "$(grep -q 'compiling souffle program' "$REPO/.axiomcode/build.log" 2>/dev/null && echo 1 || echo 0)" > "$OUT/build-seconds" 2>/dev/null || true -keep_base_graph -for l in ${READY[@]+"${READY[@]}"}; do - mkdir -p "$LANGDIR/$l/out"; swap "$OUT/$l/graph.sqlite" "$LANGDIR/$l/out/graph.sqlite" || true # a rename: atomic - rmdir "$OUT/$l" 2>/dev/null || true -done -# a language the repository no longer has (or that --lang left out) must not keep answering from an old graph -for d in "$LANGDIR"/*/; do - [ -d "$d" ] || continue; l="$(basename "$d")" - case ",$OTHER_LANGS," in *",$l,"*) ;; *) rm -rf "$d" "$REPO/.axiomcode/base/lang/$l";; esac -done -commit_base -rm -rf "$PREV" -python3 - "$OUT/graph.sqlite" <<'PY' +tiers(){ python3 -c 'import sqlite3,sys; c=sqlite3.connect(sys.argv[1]); print(", ".join(f"{t} {n}" for t, n in c.execute("SELECT tier, count(*) FROM call_edges GROUP BY tier ORDER BY 2 DESC")))' "$1"; } +# EACH LANGUAGE'S GRAPH IS PUBLISHED WHEN ITS OWN LANGUAGE IS SOLVED (#1555). The engine solves one language after +# another, and nearly all of each is that language's Souffle compile. Published only when the whole run ended, the main +# language's graph sat finished while three JavaScript files compiled: on a C# web application with a small ClientApp, +# written at 131 s and queryable at 5,304 s, and every query in between waited on the build lock. So the engine runs in +# the background and names each language as it finishes it ($PROGRESS, `all --progress`); the main one is indexed and +# pointed at right then, every other one moved into .axiomcode/lang as it comes. $PENDING lists the languages still +# being solved: ax_fresh.py reads it while this build holds the lock, so a query neither waits for them nor reads the +# graph as stale, and says which languages its answer cannot see yet. +# $PENDING is written when the main graph is published, not before: until then the graph a query reads is the previous +# one, and a rebuild in progress must keep saying so. +PROGRESS="$OUT/.progress"; PENDING="$OUT/building"; rm -f "$PROGRESS" "$PENDING"; : > "$PROGRESS" +unpend(){ grep -v "^$1 " "$PENDING" > "$PENDING.new" 2>/dev/null || true; mv -f "$PENDING.new" "$PENDING"; } +# the engine gets its own process group (set -m), so a build that is stopped stops all of it -- the Souffle compile +# included -- rather than leaving it running under the build lock it inherited +EPID="" +stop_engine(){ if [ -n "$EPID" ] && kill -0 "$EPID" 2>/dev/null; then kill -TERM -- "-$EPID" 2>/dev/null || kill -TERM "$EPID" 2>/dev/null || true; fi; rm -f "$PROGRESS" "$PENDING" "$PENDING.new"; } +trap stop_engine EXIT +trap 'exit 130' INT; trap 'exit 143' TERM HUP +# an engine older than this script has no --progress and refuses an option it does not know: it gets none, and its +# graphs are published together when it ends, as they always were +PROG=(); grep -q -- '--progress' "$ENGINE/bin/axiomcode" 2>/dev/null && PROG=(--progress "$PROGRESS") +set -m +"$ENGINE/bin/axiomcode" all "$SRC" "$OUT" --language "$SOLVE" ${PROG[@]+"${PROG[@]}"} ${LIB[@]+"${LIB[@]}"} < /dev/null > "$REPO/.axiomcode/build.log" 2>&1 & +EPID=$! +set +m +MAIN_DONE=""; QUEUED=(); READY=(); FAILED=(); DONE=" "; SEEN=0 +# THE MAIN LANGUAGE'S GRAPH BECOMES THE GRAPH. Everything a finished build did for it happens here, as soon as it is +# solved: index, stamp, file table, baseline, pointer. +publish_main(){ + # the new database: a FILE under $OUT/$LANG_ARG. Matched as `*/$LANG_ARG/*` it was any path with the language's name + # in it, the repository's own included: a repository at .../python/app matched the graph.sqlite SYMLINK, the link was + # pointed at itself, the previous graph deleted, and every verb after it reported no graph. + DB="$(find "$OUT/$LANG_ARG" -type f -name graph.sqlite 2>/dev/null | head -1)"; NOT_OTHER=(); for l in ${OTHER_LANGS//,/ }; do NOT_OTHER+=(-not -path "$OUT/$l/*"); done # never another language's graph + [ -n "$DB" ] || DB="$(find "$OUT" -type f -name graph.sqlite -not -path "$PREV/*" ${NOT_OTHER[@]+"${NOT_OTHER[@]}"} | head -1)" + [ -n "$DB" ] || { restore; echo "build produced no graph.sqlite — the graph that was there is untouched; see $REPO/.axiomcode/build.log"; exit 1; } + # A QUERY NEVER READS A HALF-BUILT GRAPH. The new database gets its symbols, refs and views BEFORE graph.sqlite points + # at it; until the pointer moves, every verb (and the hooks) keeps answering from the previous graph. Only then is the + # old one removed: a reader that opened it keeps the file it opened. + AXIOMCODE_INDEX_DB="$DB" python3 "$H/axiomcode-index" "$REPO" || { restore; echo "indexing the new graph failed — the graph that was there is untouched"; exit 1; } + stamp_meta "$DB" + echo "$STAMP" > "$OUT/stamp" + # the engine that built this graph: the next build, the background refresh above all, starts its search there (see the top) + printf '%s\n' "$ENGINE" > "$REPO/.axiomcode/engine.new" && mv -f "$REPO/.axiomcode/engine.new" "$REPO/.axiomcode/engine" || true + if [ -n "$INDEXED_TREE" ]; then echo "$INDEXED_TREE" > "$OUT/indexed-tree"; else rm -f "$OUT/indexed-tree"; fi + set_base "$INDEXED_TREE" + if [ -n "$OTHER_LANGS" ]; then S="${SOLVE#*,}"; for l in ${S//,/ }; do echo "$l $MODE"; done > "$PENDING"; fi # before the pointer: the first query may follow at once + point "$DB" || { restore; exit 1; }; rm -f "$OUT/.live.sqlite" + if [ -f "$OUT/.files.json.new" ]; then mv "$OUT/.files.json.new" "$OUT/files.json"; else rm -f "$OUT/files.json"; fi + # the graph now matches the files: record how long that took, and whether it included compiling the engine's rules (a + # one-time cost a refresh does not pay, so such a duration predicts nothing). With other languages still to come this + # is the time to the MAIN graph, which is what a query waits for. + printf '%s %s\n' "$((SECONDS - BUILD_T0))" "$(grep -q 'compiling souffle program' "$REPO/.axiomcode/build.log" 2>/dev/null && echo 1 || echo 0)" > "$OUT/build-seconds" 2>/dev/null || true + keep_base_graph + # a language the repository no longer has (or that --lang left out) must not keep answering from an old graph + for d in "$LANGDIR"/*/; do + [ -d "$d" ] || continue; l="$(basename "$d")" + case ",$OTHER_LANGS," in *",$l,"*) ;; *) rm -rf "$d" "$REPO/.axiomcode/base/lang/$l";; esac + done + commit_base + rm -rf "$PREV" + MAIN_DONE=1 + python3 - "$OUT/graph.sqlite" <<'PY' import sqlite3, sys rows = sqlite3.connect(sys.argv[1]).execute("SELECT tier, count(*) FROM call_edges GROUP BY tier ORDER BY 2 DESC").fetchall() w = max([len('tier')] + [len(t) for t, _ in rows]) print(f"{'tier':<{w}} edges") for t, n in rows: print(f"{t:<{w}} {n}") PY -python3 "$H/axiomcode-impact" --warm "$REPO" || true # export the graph's facts now (~9 s on 1,373 files): the first -# query pays it otherwise, and for the plugin that first query is inside hooks/changes.py's timeout=14, several at once. -python3 "$H/dl_program.py" || true # compile the query rules once, here where ~20 s is noise against the build: -# a later query under a caller's timeout (hooks/changes.py runs impact with timeout=14) cannot outlast the compile, so -# left to first use it is killed every time, caches nothing, and repeats on the next edit forever. -[ -n "$BASE_SAVED" ] && AXIOMCODE_GRAPH="$REPO/.axiomcode/base" python3 "$H/axiomcode-impact" --warm "$REPO" >/dev/null 2>&1 || true -for l in ${READY[@]+"${READY[@]}"}; do - echo "$l: $(python3 -c 'import sqlite3,sys; c=sqlite3.connect(sys.argv[1]); print(", ".join(f"{t} {n}" for t, n in c.execute("SELECT tier, count(*) FROM call_edges GROUP BY tier ORDER BY 2 DESC")))' "$LANGDIR/$l/out/graph.sqlite")" - AXIOMCODE_GRAPH="$LANGDIR/$l" python3 "$H/axiomcode-impact" --warm "$REPO" >/dev/null 2>&1 || true - [ -n "$BASE_SAVED" ] && [ -f "$REPO/.axiomcode/base/lang/$l/out/graph.sqlite" ] && { AXIOMCODE_GRAPH="$REPO/.axiomcode/base/lang/$l" python3 "$H/axiomcode-impact" --warm "$REPO" >/dev/null 2>&1 || true; } + python3 "$H/axiomcode-impact" --warm "$REPO" || true # export the graph's facts now (~9 s on 1,373 files): the first + # query pays it otherwise, and for the plugin that first query is inside hooks/changes.py's timeout=14, several at once. + python3 "$H/dl_program.py" || true # compile the query rules once, here where ~20 s is noise against the build: + # a later query under a caller's timeout (hooks/changes.py runs impact with timeout=14) cannot outlast the compile, so + # left to first use it is killed every time, caches nothing, and repeats on the next edit forever. + [ -n "$BASE_SAVED" ] && AXIOMCODE_GRAPH="$REPO/.axiomcode/base" python3 "$H/axiomcode-impact" --warm "$REPO" >/dev/null 2>&1 || true + [ -z "$OTHER_LANGS" ] || [ ! -s "$PENDING" ] || echo "$LANG_ARG graph ready at $OUT/graph.sqlite; still building: $(cut -d' ' -f1 "$PENDING" | paste -sd, - | sed 's/,/, /g')" +} +# EVERY OTHER LANGUAGE'S GRAPH, indexed where the engine wrote it and only then moved into place, as the main one is. +# One that did not build is NAMED: its previous graph, if any, keeps answering, and this build says it is behind. +publish_other(){ + local l="$1" LDB="$OUT/$1/graph.sqlite" + if [ "$2" = ok ] && [ -f "$LDB" ] && AXIOMCODE_INDEX_DB="$LDB" python3 "$H/axiomcode-index" "$REPO"; then + stamp_meta "$LDB" + mkdir -p "$LANGDIR/$l/out"; swap "$LDB" "$LANGDIR/$l/out/graph.sqlite" || true # a rename: atomic + rmdir "$OUT/$l" 2>/dev/null || true + READY+=("$l") + echo "$l: $(tiers "$LANGDIR/$l/out/graph.sqlite")" + AXIOMCODE_GRAPH="$LANGDIR/$l" python3 "$H/axiomcode-impact" --warm "$REPO" >/dev/null 2>&1 || true + [ -n "$BASE_SAVED" ] && [ -f "$REPO/.axiomcode/base/lang/$l/out/graph.sqlite" ] && { AXIOMCODE_GRAPH="$REPO/.axiomcode/base/lang/$l" python3 "$H/axiomcode-impact" --warm "$REPO" >/dev/null 2>&1 || true; } + else FAILED+=("$l"); fi + DONE="$DONE$l "; unpend "$l"; return 0 +} +# one language failing does not cost the others their graphs (the engine solves each on its own): only the MAIN +# language's failure fails the build, and it stops the rest, whose graphs would be dropped with it. +main_failed(){ restore; echo "build failed — the graph that was there is untouched; see $REPO/.axiomcode/build.log"; tail -5 "$REPO/.axiomcode/build.log"; exit 1; } +take(){ # one line of $PROGRESS: ok|failed [] + local l="${1%% *}" st; st="${1#* }"; st="${st%% *}" + if [ "$l" = "$LANG_ARG" ]; then + [ "$st" = ok ] || main_failed + publish_main + local q; for q in ${QUEUED[@]+"${QUEUED[@]}"}; do publish_other "${q%% *}" "${q#* }"; done; QUEUED=() + elif [ -n "$MAIN_DONE" ]; then publish_other "$l" "$st" + else QUEUED+=("$l $st"); fi # solved before the main one (it never is, but the main graph is published first) +} +drain(){ local n; n="$(wc -l < "$PROGRESS" | tr -d ' ')"; while [ "$SEEN" -lt "$n" ]; do SEEN=$((SEEN + 1)); take "$(sed -n "${SEEN}p" "$PROGRESS")"; done; } +# `running` is sampled BEFORE the drain: a line written just before the engine exited is still read +while :; do + running=1; kill -0 "$EPID" 2>/dev/null || running=0 + drain + [ "$running" = 1 ] || break + sleep 1 +done +ENGINE_RC=0; wait "$EPID" || ENGINE_RC=$?; EPID="" +drain +if [ -z "$MAIN_DONE" ]; then + # an engine that named nothing (one without --progress, or a run that failed before solving): as before, a failure + # fails the build unless it was another language's and the main graph is there + [ "$ENGINE_RC" = 0 ] || { [ -n "$OTHER_LANGS" ] && [ -f "$OUT/$LANG_ARG/graph.sqlite" ]; } || main_failed + take "$LANG_ARG ok" +fi +# a language the engine never named (the parser found no project of it) did not build, and is named below +for l in ${OTHER_LANGS//,/ }; do + case "$DONE" in *" $l "*) ;; *) publish_other "$l" ok;; esac done +rm -f "$PROGRESS" "$PENDING" if [ ${#FAILED[@]} -gt 0 ]; then for l in "${FAILED[@]}"; do if [ -f "$LANGDIR/$l/out/graph.sqlite" ]; then echo "⚠️ the $l graph did not build: $l questions are answered from its PREVIOUS graph, which predates this tree; see $REPO/.axiomcode/build.log" diff --git a/tests/README.md b/tests/README.md index 1f2d8476b..5424bfcf2 100644 --- a/tests/README.md +++ b/tests/README.md @@ -61,6 +61,10 @@ One check needs no graph and is its own script: --no-engine for the rest) python3 tests/facts_cache.py path and impact do not answer from edges an older plugin exported: the export's stamp carries a version, so an upgrade re-exports (#1402; indexes a case) + python3 tests/publish_order.py a repository in several languages is queryable when its main language is solved, not + when the last one is: the others are held back, a query answers and names what it + cannot see, a first query does not wait for them, one language alone is unchanged + (#1555; builds real graphs, needs the engine) python3 tests/tiers.py every call_edges tier the schema documents is ranked, labelled and given a certainty by the frontend, so a new tier cannot read as the weakest claim diff --git a/tests/publish_order.py b/tests/publish_order.py new file mode 100644 index 000000000..5f2dc307e --- /dev/null +++ b/tests/publish_order.py @@ -0,0 +1,234 @@ +#!/usr/bin/env python3 +"""tests/publish_order.py: a repository in several languages is queryable as soon as its main language is solved. + +The engine solves one language after another, and nearly all of each is that language's Souffle compile. The build +used to publish nothing until every language was done: on a C# web application with a small front end in another +language, the C# graph was written 131 s into the first index and the first query answered at 5,304 s (#1555). + +The other languages are held back here on purpose: the engine is wrapped by one that runs the real engine, +reports the first language it solved, and holds the rest back until the test lets them go. Nothing else is faked. + +One throwaway repository: a C# web API (the main language: most files), a small Java report job and some Python +scripts, more Python files than Java ones. + + order the engine solves the main language first, then the others by file count (python before + java), though the fixed language order puts java first + published while the other languages are held, graph.sqlite already points at the C# graph and the build is + still running + query a C# question is answered then, at once, and says which languages it cannot see yet; a Java + name is refused without waiting for its compile + rebuild a rebuild that is past its main language still has the other languages' previous graphs: a query answers + at once, as every query during a refresh does, and says which languages it answered from old graphs + finished released, the build ends with every language's graph and no trace of the pending list + first query with no graph at all, a query builds one and answers when the C# graph is ready, not when the + build ends; the build goes on and still ends with every language's graph; the answer on stdout + carries none of the build's output + main fails a main language that fails fails the build, keeps the graph that was there, and publishes nothing + control a repository in one language builds as before: no pending list, no .axiomcode/lang, no note + + python3 tests/publish_order.py [-v] +""" +import json, os, shutil, subprocess, sys, tempfile, time + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +AX = os.path.join(ROOT, 'bin', 'axiomcode') +FRESH = os.path.join(ROOT, 'plugins', 'axiomcode', 'skills', 'axiomcode', 'scripts', 'ax_fresh.py') + +CS = { + 'WebApp/WebApp.csproj': '\n \n net8.0\n \n\n', + 'WebApp/Program.cs': 'using WebApp.Services;\n\nvar builder = WebApplication.CreateBuilder(args);\nbuilder.Services.AddScoped();\nvar app = builder.Build();\napp.Run();\n', + 'WebApp/Controllers/OrdersController.cs': 'using WebApp.Services;\n\nnamespace WebApp.Controllers;\n\npublic class OrdersController\n{\n private readonly IOrderService _orders;\n\n public OrdersController(IOrderService orders) => _orders = orders;\n\n public decimal Get(int id) => _orders.Total(id);\n}\n', + 'WebApp/Services/OrderService.cs': 'namespace WebApp.Services;\n\npublic interface IOrderService\n{\n decimal Total(int orderId);\n}\n\npublic class OrderService : IOrderService\n{\n public decimal Total(int orderId) => Subtotal(orderId) * 1.2m;\n\n private decimal Subtotal(int orderId) => orderId * 10m;\n}\n', + 'WebApp/Models/Order.cs': 'namespace WebApp.Models;\n\npublic record Order(int Id, decimal Amount);\n', +} +TOOLS = { + 'tools/report/pom.xml': '\n 4.0.0\n app\n report\n 1.0\n\n', + 'tools/report/src/main/java/app/report/Formatter.java': 'package app.report;\n\npublic class Formatter {\n public String money(double x) {\n return String.format("%.2f", x);\n }\n}\n', + 'tools/report/src/main/java/app/report/ReportJob.java': 'package app.report;\n\npublic class ReportJob {\n private final Formatter formatter = new Formatter();\n\n public String render(double total) {\n return formatter.money(total);\n }\n}\n', + 'tools/scripts/totals.py': 'def sum_totals(rows):\n return sum(r["total"] for r in rows)\n', + 'tools/scripts/export.py': 'from totals import sum_totals\n\n\ndef export(rows):\n return {"sum": sum_totals(rows)}\n', + 'tools/scripts/labels.py': 'def label(x):\n return "total " + str(x)\n', +} + +# the engine, wrapped: the real `all` runs, and the first language it names is passed on the moment it is named; the +# rest only once the real run has ended AND the HOLD file is gone. So the main graph is published while the real engine +# is still solving (a cold compile of the others' rules takes as long as it takes), and the test decides when the other +# languages arrive. Without --progress (a build that publishes nothing early) it holds the same way. +SHIM = r'''#!/usr/bin/env bash +REAL="$AXIOMCODE_TEST_REAL/bin/axiomcode" +[ "${1:-}" = all ] || exec "$REAL" "$@" +prog=""; args=(); next="" +for a in "$@"; do + if [ -n "$next" ]; then prog="$a"; args+=("$a.real"); next=""; continue; fi + [ "$a" = --progress ] && next=1; args+=("$a") +done +[ -z "$prog" ] || rm -f "$prog.real" +"$REAL" "${args[@]}" & pid=$! +if [ -n "$prog" ]; then + while kill -0 "$pid" 2>/dev/null && [ ! -s "$prog.real" ]; do sleep 0.2; done + first="$(head -1 "$prog.real" 2>/dev/null)" + if [ -n "$first" ]; then + [ -n "${AXIOMCODE_TEST_FAIL_MAIN:-}" ] && first="${first%% *} failed" + echo "$first" >> "$prog" + fi +fi +wait "$pid"; rc=$? +while [ -e "$AXIOMCODE_TEST_HOLD" ]; do sleep 0.2; done +[ -z "$prog" ] || tail -n +2 "$prog.real" >> "$prog" +exit $rc +''' + + +def sh(cwd, *cmd, env=None): + return subprocess.run(cmd, cwd=cwd, capture_output=True, text=True, env=env) + + +def make(root, files): + for rel, text in files.items(): + os.makedirs(os.path.dirname(os.path.join(root, rel)), exist_ok=True) + open(os.path.join(root, rel), 'w').write(text) + for cmd in (('git', 'init', '-q'), ('git', 'add', '-A'), ('git', '-c', 'user.email=t@t', '-c', 'user.name=t', 'commit', '-qm', 'base')): + sh(root, *cmd) + + +def until(cond, seconds): + end = time.time() + seconds + while time.time() < end: + if cond(): return True + time.sleep(0.2) + return cond() + + +def main(argv): + verbose = '-v' in argv + fails = [] + + def check(ok, why, detail=''): + print(('ok ' if ok else 'FAIL ') + why + ('' if ok and not verbose or not detail else '\n ' + str(detail).strip()[-1500:].replace('\n', '\n '))) + if not ok: fails.append(why) + + work = os.path.realpath(tempfile.mkdtemp(prefix='axiomcode-publish-')) + shim = os.path.join(work, 'engine') + os.makedirs(os.path.join(shim, 'bin')) + open(os.path.join(shim, 'bin', 'axiomcode'), 'w').write(SHIM); os.chmod(os.path.join(shim, 'bin', 'axiomcode'), 0o755) + for n in ('graph', 'package.json', 'parser'): os.symlink(os.path.join(ROOT, n), os.path.join(shim, n)) + hold = os.path.join(work, 'hold') + env = dict(os.environ, AXIOMCODE_ENGINE=shim, AXIOMCODE_TEST_REAL=ROOT, AXIOMCODE_TEST_HOLD=hold, AXIOMCODE_NO_REFRESH='1') + for k in ('AXIOMCODE_LANG', 'AXIOMCODE_GRAPH', 'AXIOMCODE_GRAPH_LANG', 'AXIOMCODE_LIBRARY'): env.pop(k, None) + procs = [] + try: + repo = os.path.join(work, 'webapp'); make(repo, dict(CS, **TOOLS)) + out = os.path.join(repo, '.axiomcode', 'out'); lang = os.path.join(repo, '.axiomcode', 'lang') + graph = os.path.join(out, 'graph.sqlite') + + # ── published ───────────────────────────────────────────────────────────────────────────────────────── + open(hold, 'w').close() + b = subprocess.Popen([AX, 'index', '.'], cwd=repo, env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True); procs.append(b) + up = until(lambda: os.path.exists(graph) or b.poll() is not None, 600) + running = b.poll() is None + check(up and os.path.exists(graph) and running, + 'published: graph.sqlite points at the main (C#) graph while the other languages are still being solved', + f"graph={os.path.exists(graph)} running={running}") + log = open(os.path.join(repo, '.axiomcode', 'build.log')).read() if os.path.exists(os.path.join(repo, '.axiomcode', 'build.log')) else '' + check('▶ languages: csharp python java' in log, + 'order: the main language is solved first, then the others by file count (python 3 before java 2)', + [l for l in log.splitlines() if 'languages' in l]) + if not (up and running): return 1 + until(lambda: os.path.exists(os.path.join(out, 'building')), 30) + + # ── query ───────────────────────────────────────────────────────────────────────────────────────────── + t0 = time.time(); q = sh(repo, AX, 'impact', 'OrderService.Total', '.', env=env); dt = time.time() - t0 + check(q.returncode == 0 and 'OrdersController.Get' in q.stdout and b.poll() is None and dt < 60, + 'query: a C# question is answered while the other languages are held, without waiting for them', + f"{dt:.1f}s\n{q.stdout}\n{q.stderr}") + check('the python, java graphs are still being built' in q.stderr, + 'query: the answer names the languages whose graphs it cannot see yet', q.stderr) + live = dict(env); live.pop('AXIOMCODE_NO_REFRESH') + t0 = time.time(); q = sh(repo, AX, 'impact', 'OrderService.Total', '.', env=live); dt = time.time() - t0 + check(q.returncode == 0 and 'OrdersController.Get' in q.stdout and 'still being built' in q.stderr and dt < 60 and b.poll() is None, + 'query: with the refresher on, the same answer and note, and no wait for a refresh', f"{dt:.1f}s\n{q.stderr}") + st = json.loads(sh(repo, sys.executable, FRESH, 'status', '.', '--json', env=env).stdout or '{}') + check(st.get('state') == 'building' and st.get('pending') == ['python', 'java'] and st.get('first') is True, + 'query: status says building, with the pending languages in solve order, on a first build', json.dumps(st)) + t0 = time.time(); r = sh(repo, AX, 'impact', 'Formatter.money', '.', env=env); dt = time.time() - t0 + check(r.returncode != 0 and 'Formatter.money' in r.stdout + r.stderr and b.poll() is None and dt < 60, + 'query: a Java name is refused at once, not after the Java solve', f"{dt:.1f}s\n{r.stdout}\n{r.stderr}") + + # ── finished ────────────────────────────────────────────────────────────────────────────────────────── + os.remove(hold) + bout, _ = b.communicate(timeout=5400) + have = {l: os.path.exists(os.path.join(lang, l, 'out', 'graph.sqlite')) for l in ('python', 'java')} + check(b.returncode == 0 and all(have.values()) and not os.path.exists(os.path.join(out, 'building')) and not os.path.exists(os.path.join(out, '.progress')), + 'finished: the build ends with every language\'s graph, and the pending list is gone', bout + json.dumps(have)) + check(bout.index('csharp graph ready') < bout.index('python:') < bout.index('java:') and 'still building: python, java' in bout, + 'finished: the build says when the main graph is ready, then names each language as it is published', bout) + r = sh(repo, AX, 'impact', 'Formatter.money', '.', env=env) + check(r.returncode == 0 and 'ReportJob.render' in r.stdout and 'still being built' not in r.stderr, + 'finished: the Java name is answered from its own graph, with no note', r.stdout + r.stderr) + + # ── rebuild ─────────────────────────────────────────────────────────────────────────────────────────── + api = os.path.join(repo, 'tools/scripts/totals.py') + open(api, 'a').write('\n\ndef added_later():\n return 1\n') + open(hold, 'w').close() + b = subprocess.Popen([AX, 'index', '.'], cwd=repo, env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True); procs.append(b) + until(lambda: os.path.exists(os.path.join(out, 'building')) or b.poll() is not None, 600) + t0 = time.time(); q = sh(repo, AX, 'impact', 'OrderService.Total', '.', env=live); dt = time.time() - t0 + check(q.returncode == 0 and 'OrdersController.Get' in q.stdout and dt < 60 and b.poll() is None + and 'its python, java code comes from the previous graph' in q.stderr, + 'rebuild: past its main language, a query answers at once and says the other languages\' code is from their previous graphs', + f"{dt:.1f}s\n{q.stderr}") + os.remove(hold); b.communicate(timeout=5400) + r = sh(repo, AX, 'impact', 'added_later', '.', env=env) + check(b.returncode == 0 and r.returncode == 0 and 'totals.py' in r.stdout, 'rebuild: released, the edit is in the Python graph', r.stdout + r.stderr) + sh(repo, 'git', 'checkout', '-q', '--', '.') + + # ── main fails ──────────────────────────────────────────────────────────────────────────────────────── + before = sh(repo, AX, 'impact', 'OrderService.Total', '.', env=env).stdout + model = os.path.join(repo, 'WebApp/Models/Order.cs') + open(model, 'a').write('\npublic record Line(int Qty);\n') # a change, so the index rebuilds + f = sh(repo, AX, 'index', '.', env=dict(env, AXIOMCODE_TEST_FAIL_MAIN='1')) + after = sh(repo, AX, 'impact', 'OrderService.Total', '.', env=env) + check(f.returncode != 0 and 'build failed' in f.stdout and after.returncode == 0 and after.stdout == before + and not os.path.exists(os.path.join(out, 'building')), + 'main fails: the build fails, and the graph that was there answers exactly as before', f.stdout + f.stderr) + sh(repo, 'git', 'checkout', '-q', '--', '.') + + # ── first query ─────────────────────────────────────────────────────────────────────────────────────── + shutil.rmtree(os.path.join(repo, '.axiomcode')) + open(hold, 'w').close() + t0 = time.time(); q = sh(repo, AX, 'impact', 'OrderService.Total', '.', env=env); dt = time.time() - t0 + held = os.path.exists(hold) and not os.path.exists(os.path.join(lang, 'java')) + check(q.returncode == 0 and 'OrdersController.Get' in q.stdout and held, + 'first query: with no graph, the answer comes when the C# graph is ready, while the other languages are held', + f"{dt:.1f}s held={held}\n{q.stdout}\n{q.stderr}") + check('tier ' not in q.stdout and 'indexed ' not in q.stdout and 'indexed ' in q.stderr, + 'first query: the build\'s output goes to stderr, and the answer on stdout is only the answer', q.stdout[:600]) + check('still being built' in q.stderr, 'first query: the answer names the languages it cannot see yet', q.stderr[-600:]) + os.remove(hold) + done = until(lambda: all(os.path.exists(os.path.join(lang, l, 'out', 'graph.sqlite')) for l in ('python', 'java')) + and not os.path.exists(os.path.join(out, 'building')), 600) + check(done, 'first query: the build goes on after the answer and ends with every language\'s graph', + open(os.path.join(repo, '.axiomcode', 'index.log')).read()[-800:] if os.path.exists(os.path.join(repo, '.axiomcode', 'index.log')) else '') + + # ── control ─────────────────────────────────────────────────────────────────────────────────────────── + one = os.path.join(work, 'api-only'); make(one, CS) + c = sh(one, AX, 'index', '.', env=env) + q = sh(one, AX, 'impact', 'OrderService.Total', '.', env=env) + check(c.returncode == 0 and 'still building' not in c.stdout and not os.path.exists(os.path.join(one, '.axiomcode', 'lang')) + and not os.path.exists(os.path.join(one, '.axiomcode', 'out', 'building')) and c.stdout.rstrip().endswith('graph at ' + os.path.join(one, '.axiomcode', 'out', 'graph.sqlite')), + 'control: a repository in one language builds as it did, with no pending list and no other graph', c.stdout + c.stderr) + check(q.returncode == 0 and 'OrdersController.Get' in q.stdout and 'graph refresh' not in q.stderr, + 'control: its answer carries no note', q.stdout + q.stderr) + finally: + try: os.remove(hold) + except OSError: pass + for p in procs: + if p.poll() is None: p.kill() + time.sleep(1) + shutil.rmtree(work, ignore_errors=True) + print(f"\n{'FAILED: ' + str(len(fails)) if fails else 'all passed'}") + return 1 if fails else 0 + + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:]))