Skip to content

Commit 2bfffca

Browse files
swapnilpaliwal-sdswapnilpaliwal-sd
authored
Merge pull request #1931 from AxiomCodeAI/fix/e2e-queries-019
e2e-queries: check 0.1.9's answers and verbs (#1930)
2 parents 7e22334 + a650342 commit 2bfffca

1 file changed

Lines changed: 30 additions & 22 deletions

File tree

‎.github/scripts/e2e-queries.sh‎

Lines changed: 30 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,8 @@
88
# calls entry(), and `expect` names the four in that language's spelling. The project is
99
# copied to <work-dir>, committed, indexed, and asked:
1010
#
11-
# impact LEAF lists HELPER, the method that calls it
12-
# path ENTRY LEAF reached, through HELPER
11+
# impact LEAF lists HELPER, the method that calls it, as a resolved place with its code
12+
# path ENTRY LEAF a chain through HELPER, every hop re-checked
1313
# path ENTRY LEAF the same from the shipped Datalog programs (AXIOMCODE_DATALOG=1)
1414
# path … --every at least one route listed, both backends
1515
# context LEAF names LEAF
@@ -20,11 +20,10 @@
2020
# impact, path (Datalog) with os.symlink refused, as for an unelevated Windows user (#1363)
2121
# impact, path, context with os.exec* ending the caller at once, as Windows' exec does (#1640)
2222
# Windows: impact run from a directory holding a git.exe, python.exe and py.exe (#1332)
23-
# graph --out, help impact, --version
23+
# help impact, --version
2424
# leaf's 41 becomes 42, then
25-
# changed names LEAF
26-
# changed --impact names HELPER as reached
27-
# test-impact selects TEST
25+
# impact (no name) names LEAF as edited, and HELPER as reached
26+
# tests selects TEST, with the command that runs it
2827
#
2928
# Running is not passing: each answer has to contain what the project makes true, and no
3029
# answer may carry a Python traceback.
@@ -52,11 +51,21 @@ must() { # <extended regex> the last answer has to match, and why
5251
grep -Eq -- "$1" "$R/.q.log" || { sed 's/^/ /' "$R/.q.log" | head -25; fail "$LABEL: expected /$1/ ($2)"; }
5352
echo " ok $LABEL — $2"
5453
}
54+
# The answers are the front door's: numbered places, each with the code of the function it sits in.
55+
# A caller is right when a [resolved] place's marked line is HELPER's; a chain when a hop names HELPER.
56+
calls_ok() { # <why>: HELPER is a resolved place in the last answer
57+
# HELPER is named by the place's tag ([resolved · helper], where its first line is not the one marked) or on the
58+
# marked line itself (where the declaration and the call share it)
59+
must "^1\. .*\[resolved" "$1 (a resolved place)"
60+
must "\[resolved[^]]*\b$HELPER\b[^]]*\]|→ +[0-9]+ .*\b$HELPER\b" "$1 (the place is $HELPER)"
61+
}
62+
path_ok() { # <why>: the last path answer is a chain through HELPER whose hops were re-checked
63+
must "hop [0-9]+/[0-9]+ [^]]*\b$HELPER\b" "$1 (a hop through $HELPER)"; must "^verified: ✓" "$1 (every hop re-checked)"
64+
}
5565
DL="AXIOMCODE_DATALOG=1"
56-
run "" impact "$LEAF"; must "\[resolved\] ([A-Za-z_.]*\.)?$HELPER .*calls it" "$HELPER is a resolved caller"
66+
run "" impact "$LEAF"; calls_ok "$HELPER is a resolved caller"
5767
for e in "" "$DL"; do
58-
run "$e" path "$ENTRY" "$LEAF"; must "reached" "a chain exists"
59-
must "→ .*([A-Za-z_.]*\.)?$HELPER " "the chain goes through $HELPER"
68+
run "$e" path "$ENTRY" "$LEAF"; path_ok "the chain goes through $HELPER"
6069
run "$e" path "$ENTRY" "$LEAF" --every; must "[0-9]+ hop\(s\): .*$HELPER.* → .*$LEAF" "a route through $HELPER to $LEAF is listed"
6170
done
6271
run "" context "$LEAF"; must "^ +([A-Za-z_.]*\.)?$LEAF +" "$LEAF is an entry point"
@@ -75,7 +84,7 @@ read -r QLEAF LEAF_AT < <(node -e '
7584
[ -n "${QLEAF:-}" ] || fail "the graph has no declaration named $LEAF"
7685
DOTTED="$(printf '%s' "$QLEAF" | sed -e 's/::/./g' -e 's/[\/\\#$]/./g' -e 's/\.\.*/./g' -e 's/^\.//' -e 's/\.$//')"
7786
for t in "$QLEAF" "$DOTTED" "$LEAF_AT"; do
78-
run "" impact "$t"; must "\[resolved\] ([A-Za-z_.]*\.)?$HELPER .*calls it" "$HELPER is a resolved caller"
87+
run "" impact "$t"; calls_ok "$HELPER is a resolved caller"
7988
done
8089
run "" impact "$LEAF" --json; must "\"$HELPER\"|[.#/]$HELPER\"" "the JSON names $HELPER"
8190
node -e 'JSON.parse(require("fs").readFileSync(process.argv[1], "utf8"))' "$(native_path "$R/.q.log")" \
@@ -86,8 +95,8 @@ run "" path '*' "$LEAF"; must "([A-Za-z_.]*\.)?$ENTRY\b"
8695
NOSYM="$R.nosymlink"; mkdir -p "$NOSYM"
8796
printf 'import os\ndef _deny(*a, **k):\n raise OSError(1314, "A required privilege is not held by the client")\nos.symlink = _deny\n' > "$NOSYM/sitecustomize.py"
8897
NS="PYTHONPATH=$(native_path "$NOSYM")"
89-
run "$NS" impact "$LEAF"; must "\[resolved\] ([A-Za-z_.]*\.)?$HELPER .*calls it" "$HELPER is a caller without symlinks"
90-
run "$NS $DL" path "$ENTRY" "$LEAF"; must "reached" "the Datalog path answers without symlinks"
98+
run "$NS" impact "$LEAF"; calls_ok "$HELPER is a caller without symlinks"
99+
run "$NS $DL" path "$ENTRY" "$LEAF"; path_ok "the Datalog path answers without symlinks"
91100

92101
# ── os.exec* that does not replace the process, as on Windows (#1640) ─────────────────────────
93102
# Windows' exec starts the program as a NEW process and ends the caller at once with status 0, so a verb handed over
@@ -111,8 +120,8 @@ PY
111120
SCR="$(cd "$(dirname "$bin")/../@axiomcode/code-graph/plugins/axiomcode/skills/axiomcode/scripts" 2>/dev/null && pwd)"
112121
[ -n "$SCR" ] || fail "the installed package has no plugin scripts next to $bin"
113122
NX="PYTHONPATH=$(native_path "$NOEXEC") E2E_AX_SCRIPTS=$(native_path "$SCR")"
114-
run "$NX" impact "$LEAF"; must "\[resolved\] ([A-Za-z_.]*\.)?$HELPER .*calls it" "$HELPER is a caller where exec does not replace the process"
115-
run "$NX" path "$ENTRY" "$LEAF"; must "reached" "path answers where exec does not replace the process"
123+
run "$NX" impact "$LEAF"; calls_ok "$HELPER is a caller where exec does not replace the process"
124+
run "$NX" path "$ENTRY" "$LEAF"; path_ok "path answers where exec does not replace the process"
116125
run "$NX" context "$LEAF"; must "^ +([A-Za-z_.]*\.)?$LEAF +" "context answers where exec does not replace the process"
117126

118127
# ── Windows: a git.exe / python.exe / py.exe in the working directory is not the one run (#1332) ──
@@ -122,21 +131,20 @@ if [ -n "$win" ]; then
122131
LABEL="impact $LEAF from a directory holding git.exe, python.exe and py.exe"
123132
( cd "$TRAP" && "$bin" impact "$LEAF" "$R" ) > "$R/.q.log" 2>&1 \
124133
|| { sed 's/^/ /' "$R/.q.log" | head -25; fail "$LABEL: rc=$?"; }
125-
must "\[resolved\] ([A-Za-z_.]*\.)?$HELPER .*calls it" "the programs in the working directory were not run"
134+
calls_ok "the programs in the working directory were not run"
126135
fi
127136

128-
# ── the rest of the CLI: graph, help, --version ───────────────────────────────────────────────
129-
run "" graph --out "$R/.graph.html"
130-
[ -s "$R/.graph.html" ] || fail "graph --out wrote no page"; echo " ok graph --out — $(wc -c < "$R/.graph.html" | tr -d ' ') bytes"
137+
# ── the rest of the CLI: help, --version (0.1.9's entry point has no graph verb) ───────────────
131138
run "" help impact; must "axiomcode impact" "help describes impact"
132139
want="$(node -p 'require(process.argv[1]).version' "$(native_path "$(dirname "$bin")/../@axiomcode/code-graph/package.json")" 2>/dev/null)"
133140
run "" --version; must "^${want//./\\.}\$" "--version is the installed version ($want)"
134141

135142
# a real edit to leaf's body: what changed, and which tests have to run for it
136143
sed 's/41/42/' "$R/$LEAF_FILE" > "$R/.edit" && mv "$R/.edit" "$R/$LEAF_FILE"
137144
git -C "$R" diff --quiet && fail "the edit to $LEAF_FILE changed nothing"
138-
run "" changed; must "([A-Za-z_.]*\.)?$LEAF\b" "$LEAF is reported changed"
139-
run "" changed --impact; must "([A-Za-z_.]*\.)?$HELPER\b" "the edit hook's answer reaches $HELPER"
140-
run "" test-impact; must "^tests to run: [1-9]" "a test reaches the change through the graph"
141-
must "^ +\S*$TEST\S* +\(" "$TEST is the test selected"
145+
run "" impact; must "^your edits: .*\b$LEAF\b" "$LEAF is reported edited"
146+
calls_ok "the edit reaches $HELPER"
147+
run "" tests; must "\[test[^]]*\].*" "a test reaches the change through the graph"
148+
must "\b$TEST\b" "$TEST is the test selected"
149+
must "^run: " "the command that runs it"
142150
echo "e2e queries: every verb answered correctly for $(basename "$fx")"

0 commit comments

Comments
 (0)