diff --git a/CLAUDE.md b/CLAUDE.md
index 2c38f04cc1..d1a28ec9a2 100644
--- a/CLAUDE.md
+++ b/CLAUDE.md
@@ -321,7 +321,7 @@ Most layout is obvious from `ls`. The non-obvious ones:
- `utils/detect-dupe/` (in-repo dupe finder) and `utils/find-dupe/` (Claude judge; needs `daspkg install --root utils/find-dupe` + `ANTHROPIC_API_KEY`) - both also MCP tools
- `utils/mcp/`, `utils/daslang-live/`, `utils/daspkg/` - in-tree tools (most also have skills)
- `tutorials/language/` (language tour) vs `tutorials//` (per-area) - never put tutorials in `modules//tutorial/`
-- **Array/dim/vector indexing lives across 5 tiers** - bounds checks, neg-index detection and width-aware bounds usually need parallel edits in all: AOT C++ (`include/daScript/simulate/aot.h`), interpreter non-fused (`include/daScript/simulate/runtime_array.h` + `simulate_nodes.h`), interpreter fused (`src/simulate/simulate_fusion_at_array.cpp` + `simulate_fusion_at.cpp`), JIT (`modules/dasLLVM/daslib/llvm_jit.das`), AST const-fold (`src/ast/ast_simulate.cpp`). Debug builds bypass the fused permutations - a fused-path-only fix passes Release CI and trips Debug-ARM. Bump `LLVM_JIT_CODEGEN_VERSION` in `modules/dasLLVM/daslib/llvm_jit_run.das` after JIT emitter/ABI changes to invalidate cached `.dll`s (the cache hash also folds per-function AOT hashes, so AST-level changes self-invalidate)
+- **Array/dim/vector indexing lives across 5 tiers** - bounds checks, neg-index detection and width-aware bounds usually need parallel edits in all: AOT C++ (`include/daScript/simulate/aot.h`), interpreter non-fused (`include/daScript/simulate/runtime_array.h` + `simulate_nodes.h`), interpreter fused (`src/simulate/simulate_fusion_at_array.cpp` + `simulate_fusion_at.cpp`), JIT (`modules/dasLLVM/daslib/llvm_jit.das`), AST const-fold (`src/ast/ast_simulate.cpp`). Debug builds bypass the fused permutations - a fused-path-only fix passes Release CI and trips Debug-ARM. Bump `LLVM_JIT_CODEGEN_VERSION` in `modules/dasLLVM/daslib/llvm_jit_plan.das` after JIT emitter/ABI changes to invalidate cached `.dll`s (the cache hash also folds per-function AOT hashes, so AST-level changes self-invalidate)
## MCP Server (AI Tool Integration)
diff --git a/daslib/ARCHITECTURE.md b/daslib/ARCHITECTURE.md
index 9dba5c40eb..ae4c6e5510 100644
--- a/daslib/ARCHITECTURE.md
+++ b/daslib/ARCHITECTURE.md
@@ -232,15 +232,29 @@ Three companions carry a concern each; a section number is unique across all fou
share one macro context; `tune_status` and `log_tune_status` are shells of the same kind.
- **The shells decide with `typeinfo module_exists(llvm_tune)`, a pair with the trait's C++ half
in `src/ast/ast_infer_type.cpp` (`module_exists`): visible from the compiling module, which is
- what the `require ?llvm llvm/daslib/llvm_tune public` line above them decided.** Nothing fails
+ what the `require [tune_framework] public` line above them decided.** Nothing fails
when the two sides drift - `tests/language/optional_require.das` pins the trait's visibility
half on fixtures, and `tests/daslib/test_tune_shells.das` is tier-agnostic by design - so the pair
is recorded here: a trait that answered off the process registry instead would take the
framework arm inside a tool's nested compile, where the framework is not mounted.
-- **The `llvm/daslib/*` files resolve in every build that carries `modules/dasLLVM/`; only the
- witness decides what `?llvm` and `module_exists(llvm_tune)` answer.** The framework's das files
- compile without their C++ module, so a direct `require llvm/daslib/llvm_tune` in a build without
- dasLLVM compiles with the shells inert and no `_variants()` registry - a program that
- reads one is framework-only and says so with that direct require. `daslib/just_in_time` keeps
- its direct require for the opposite reason: a static host that never registered the witness
- still runs the JIT through the LLVM library, and the guard would switch it off.
+- **The `llvm/daslib/*` files resolve in every build that carries `modules/dasLLVM/`; only a
+ build configured with dasLLVM joins the `tune_framework` group with `llvm_tune` - the `llvm`
+ witness module's constructor for a static host, the descriptor's row guarded on `llvm` for a
+ DLL build - so the group, and with it `module_exists(llvm_tune)`, follows the configure and
+ never the das files on disk.** `llvm_code` is not a member: `llvm_tune` requires
+ it `public`, so every requirer of the group sees the `[llvm_code]` annotation through it, as
+ every requirer of `llvm_tune` always did. The framework's das files compile without their C++ module, so a direct
+ `require llvm/daslib/llvm_tune` in a build without dasLLVM compiles with the shells inert and no
+ `_variants()` registry - a program that reads one is framework-only and says so with that
+ direct require. `daslib/just_in_time` keeps its direct require for the opposite reason: a static
+ host that never registered the witness still runs the JIT through the LLVM library, and the
+ group would switch it off.
+
+## 38. cross_context
+
+- **A function reached by name in the context `macro_context_of` answers is `[export]`ed, a pair
+ with `Program::markMacroSymbolUse` (`src/ast/ast_export.cpp`), whose roots are `exports`, `init`,
+ `shutdown` and `macroInit` - every other function is dropped from the macro context.** Nothing
+ fails when one side moves alone: the tests name an `[export]`ed function, so a narrower or wider
+ C++ root set passes them. The module itself is not the caller's dependency (`src/ast/ARCHITECTURE.md`
+ sec.3): its macro context is the caller's only handle on it.
diff --git a/daslib/ARCHITECTURE_EMIT.md b/daslib/ARCHITECTURE_EMIT.md
index 23819ff21d..cc91418de8 100644
--- a/daslib/ARCHITECTURE_EMIT.md
+++ b/daslib/ARCHITECTURE_EMIT.md
@@ -87,12 +87,12 @@ Companion to `ARCHITECTURE.md` in this folder; section numbers are unique across
runs it, collecting on its first tick.
- **There is no `!stopFlags` guard between `[init]` calls** - a panic propagates out of the
ctor instead of soft-stopping the sequence.
-- **Global init order is fixed by `var->index` assignment**: `StandaloneContextGen`'s
+- **Global init order is fixed by the program's variable index table**: `StandaloneContextGen`'s
`preVisitGlobalLet` emits required modules' globals via ordered `for_each_module`
before the adapter walks the entry module's own - correct only because
- `ast_allocate_stack.cpp` assigns `var->index` through the same dependency-first
- module order with the entry module last, and `runInitScript` executes ascending
- index. `[init]` function order is not re-derived at all: the emitter reads the
+ `ast_allocate_stack.cpp` fills `Program::variableIndices` (read back as
+ `variable_index(program, var)`) through the same dependency-first module order with the
+ entry module last, and `runInitScript` executes ascending index. `[init]` function order is not re-derived at all: the emitter reads the
simulated context's list through rtti `for_each_init_function`, so the C++ late-init
sort stays the single source of truth.
- **The TU holds every used function of every module.** A standalone context is one
diff --git a/daslib/ARCHITECTURE_LINQ.md b/daslib/ARCHITECTURE_LINQ.md
index 63a24a98a9..e1b32627c6 100644
--- a/daslib/ARCHITECTURE_LINQ.md
+++ b/daslib/ARCHITECTURE_LINQ.md
@@ -178,9 +178,14 @@ Companion to `ARCHITECTURE.md` in this folder; section numbers are unique across
`linq_fold_xml`. `linq_fold_sql` is the exception - it recognizes a `[sql_table]` source
and hands the whole chain to the `_sql` macro, so it has no adapter and no emit. A source
file requires `linq_fold_common` and never a sibling source.
-- **`try_splice_patterns` tries recognizers in a fixed order** - sql, decs, xml (behind the
- pugixml `static_if`), json, table - and the array arm runs last with no recognizer of its
- own: it is what claims a chain nobody else claimed.
+- **`try_splice_patterns` tries recognizers in a fixed order** - sql, decs, the registered
+ `linq_fold_sources` in registration order, json, table - and the array arm runs last with
+ no recognizer of its own: it is what claims a chain nobody else claimed. A source outside
+ daslib joins the `linq_fold_source` group from its descriptor and from its C++ module's
+ constructor, and registers a
+ `LinqFoldSourceRecognizer` from its `register_linq_fold_source` entry, which
+ `register_all_linq_fold_rows` calls on every member; pugixml's `linq_fold_xml` is the one
+ such source.
- **The adapter contract is four abstract methods; everything else on the base has a
default.** `bind_name`, `element_type`, `wrap_source_loop` and `wrap_invoke` are abstract,
so every source answers them. The per-operation hooks (`emit_loop_or_count`,
diff --git a/daslib/ARCHITECTURE_LINT.md b/daslib/ARCHITECTURE_LINT.md
index 03a9adb8ad..abcbdad05a 100644
--- a/daslib/ARCHITECTURE_LINT.md
+++ b/daslib/ARCHITECTURE_LINT.md
@@ -234,3 +234,15 @@ Companion to `ARCHITECTURE.md` in this folder; section numbers are unique across
- **C++ mirror pairs**: the STYLE024/025 unsafe map <-> infer's `unsafe_*` rules;
`style036_inert_contract` <-> infer's contract clearing. Nothing fails when one side
moves alone.
+
+### 4.1 STYLE029/030 over a require group {#style-lint-require-group}
+
+- **The source line at a declaration's (file, line, column) key, read for the `[...]`
+ spelling after the keyword and its guard, is what tells a group apart from a require written
+ by hand.** A group's members share the group line's key, but a group with one member has one
+ declaration, exactly like a plain require, so the key alone cannot say; both rules judge
+ members and then keep, suppress and report per key.
+- **A declaration's verdict is settled only after every member sharing its key has been
+ judged**, so `st030_report_requires` records verdicts in one walk and reports them in a
+ later one. Reporting inside the deciding walk would fire on a member judged ahead of the
+ used member that keeps the line.
diff --git a/daslib/REVIEW.md b/daslib/REVIEW.md
index 0566dff7fb..27db867a1a 100644
--- a/daslib/REVIEW.md
+++ b/daslib/REVIEW.md
@@ -40,13 +40,13 @@ a stack, never in a bare scalar.** A saved value kept in a scalar survives into
construct's exit path and unbalances it.
**A diff that adds or changes a daslib fact - code or a `//!` contract - whose truth is
-decided by a C++-side definition, with no test, lint, or compile error failing when the two
-sides no longer match, records the pair, naming both sides, in whichever `ARCHITECTURE*.md` in
-this folder holds the daslib file's section - adding that section to `ARCHITECTURE.md` when no
-doc holds one.**
+decided by a definition outside this folder - a C++ definition, or another module's descriptor
+- with no test, lint, or compile error failing when the two sides no longer match, records the
+pair, naming both sides, in whichever `ARCHITECTURE*.md` in this folder holds the daslib file's
+section - adding that section to `ARCHITECTURE.md` when no doc holds one.**
-**When a diff changes one side of a recorded daslib/C++ pair so the two no longer match, it
-changes the other side in the same diff.**
+**When a diff changes one side of a recorded pair so the two no longer match, it changes the
+other side in the same diff.**
**A diff that renames, deletes, or changes the behavior of a daslib name that an
`ARCHITECTURE*.md` in this folder spells updates that entry in the same change.** The doc is
@@ -246,6 +246,12 @@ signed sum wraps, the bounds test passes, and the walk leaves the array silently
and emit only the taken arm.** Nothing folds at macro-application time, so a generated
`if ($v(flag))` keeps its dead arm and type-checks it.
+**A diff that adds a function to `cross_context.das` wrapping `invoke_in_context` - taking the
+`unsafe` and the `addr` off the caller - is a defect; the caller writes the call itself.** The
+wrapper's signature checks nothing the call does not: the target is found by name and argument
+count, and a pointer crosses into a context that holds its lock for the whole run, so the
+`unsafe` belongs at the site that hands the pointer over.
+
**A diff that adds or changes a buffer-I/O overload returns before taking `addr(buf[0])` on
an empty buffer.** The address is out of bounds and the call sits inside `unsafe`, so an
overload without the guard passes every non-empty test.
diff --git a/daslib/REVIEW_LINQ.md b/daslib/REVIEW_LINQ.md
index f44af7d7a9..fa2a79bf7f 100644
--- a/daslib/REVIEW_LINQ.md
+++ b/daslib/REVIEW_LINQ.md
@@ -71,8 +71,9 @@ a projected adapter is a defect - skip the row instead.** The row's emit assumes
element, so running it on the un-projected adapter orders or dedups raw rows - a wrong result,
not a missed splice.
-**A diff that reorders the `build_*_rows()` calls in `register_all_linq_fold_rows` without
-naming the chains whose matching arm changes is a defect.** Rows land in `splice_patterns` in
+**A diff that reorders the registration calls in `register_all_linq_fold_rows` - the
+`build_*_rows()` calls, the group call that registers the sources - without naming the chains
+whose matching arm changes is a defect.** Rows land in `splice_patterns` in
call order, the walker takes the first match, and that registrar is the only place the order
exists.
diff --git a/daslib/aot_cpp.das b/daslib/aot_cpp.das
index 9ce915f1f9..0a9cebdea3 100644
--- a/daslib/aot_cpp.das
+++ b/daslib/aot_cpp.das
@@ -626,6 +626,7 @@ class UseTypeMarker : AstVisitor {
useHandles : table;
useModules : table;
inUsedCode : bool = false;
+ program : ProgramPtr;
def override preVisitTypeDecl(typeDecl : TypeDeclPtr) {
}
@@ -635,7 +636,7 @@ class UseTypeMarker : AstVisitor {
}
def override preVisitFunction(fun : FunctionPtr) {
- inUsedCode = fun.flags.used;
+ inUsedCode = is_used(program, fun);
}
def override visitFunction(var fun : FunctionPtr) : FunctionPtr {
inUsedCode = false;
@@ -666,7 +667,7 @@ class UseTypeMarker : AstVisitor {
mark(variable._type);
}
def override preVisitGlobalLetVariable(variable : Variable?; lastArg : bool) {
- inUsedCode = variable.flags.used;
+ inUsedCode = is_used(program, variable);
mark(variable._type);
}
def override visitGlobalLetVariable(var variable : VariablePtr; lastArg : bool) : VariablePtr {
@@ -1571,7 +1572,7 @@ class public CppAot : AstVisitor {
}
def override preVisitGlobalLetVariable(variable : Variable?; lastArg : bool) {
write(*ss, tabs());
- if (!variable.flags.used) {
+ if (!is_used(program, variable)) {
write(*ss, "/* ");
}
if (variable.flags.global_shared) {
@@ -1587,7 +1588,7 @@ class public CppAot : AstVisitor {
}
def override visitGlobalLetVariable(variable : VariablePtr; last : bool) {
write(*ss, ";");
- if (!variable.flags.used) {
+ if (!is_used(program, variable)) {
write(*ss, " */");
}
write(*ss, "/*{variable.name}*/\n");
@@ -4211,7 +4212,7 @@ def private aotEmitStruct(ps : Structure?;
def public dumpDependencies(program : ProgramPtr; var aotVisitor : CppAot?; prune_to_used : bool = false) {
//! Writes forward declarations and dependent type/function definitions for AOT output.
buildStructEnumCollisions(program);
- var utm = new UseTypeMarker();
+ var utm = new UseTypeMarker(program := program);
make_visitor(*utm) $(adapter) {
visit(program, adapter);
if (prune_to_used) { // a standalone context emits the foreign functions too
@@ -4327,13 +4328,13 @@ class ArgsConverter : AstVisitor {
-def collectUsedFunctions(modules : array; totalFunctions : int; this_module : Module?; all_modules : bool; is_all : bool = false) : array {
+def collectUsedFunctions(program : ProgramPtr; modules : array; totalFunctions : int; this_module : Module?; all_modules : bool; is_all : bool = false) : array {
var fnn : array;
fnn.reserve(totalFunctions);
for (pm in modules) {
pm |> for_each_module_function($(pfun) {
if ((!all_modules && pfun._module != this_module)
- || pfun.index < 0 || !pfun.flags.used) return ;
+ || function_index(program, pfun) < 0 || !is_used(program, pfun)) return ;
if (!is_all) {
if (pfun.flags.builtIn || pfun.flags.noAot) return ;
}
@@ -4352,7 +4353,7 @@ def public foreignUsedFunctions(program : ProgramPtr; before_no_aot_marking : bo
program.get_ptr() |> for_each_module_no_order($(pm) {
if (pm == thisModule) return
pm |> for_each_module_function($(pfun) {
- if (pfun.index < 0 || !pfun.flags.used || pfun.flags.builtIn || pfun.moreFlags.isTemplate
+ if (function_index(program, pfun) < 0 || !is_used(program, pfun) || pfun.flags.builtIn || pfun.moreFlags.isTemplate
|| (!before_no_aot_marking && pfun.flags.noAot)) return
foreign |> push(pfun)
})
@@ -4366,7 +4367,7 @@ def public collectProgramUsedFunctions(program : ProgramPtr; all_modules : bool;
program.get_ptr() |> for_each_module_no_order($(mod) {
modules.push(mod);
})
- return collectUsedFunctions(modules, program.totalFunctions, program.getThisModule, all_modules, is_all)
+ return collectUsedFunctions(program, modules, program.totalFunctions, program.getThisModule, all_modules, is_all)
}
def registerAotCpp(var logs : StringBuilderWriter?; program : ProgramPtr; var context : Context; cross_platform; headers, all_modules : bool = false) {
@@ -4435,14 +4436,15 @@ def public setAotHashes(program : ProgramPtr; ctx : Context) {
var fni = 0;
program.get_ptr() |> for_each_module_no_order($(mod) {
mod |> for_each_module_function($(pfun) {
- if (pfun.index < 0 || !pfun.flags.used) return ;
- pfun.hash = get_function_hash_by_id(pfun, pfun.index, unsafe(reinterpret(ctx)));
+ let pindex = function_index(program, pfun)
+ if (pindex < 0 || !is_used(program, pfun)) return ;
+ pfun.hash = get_function_hash_by_id(pfun, pindex, unsafe(reinterpret(ctx)));
fni++;
})
});
program.get_ptr() |> for_each_module_no_order($(mod) {
mod |> for_each_module_function($(pfun) {
- if (pfun.index < 0 || !pfun.flags.used) return ;
+ if (function_index(program, pfun) < 0 || !is_used(program, pfun)) return ;
pfun.aotHash = get_function_aot_hash(pfun);
});
});
@@ -4481,12 +4483,12 @@ def private collectUsedModules(program : ProgramPtr) : table {
var used : table
program.get_ptr() |> for_each_module_no_order($(pm) {
pm |> for_each_module_function($(pfun) {
- if (pfun.index >= 0 && pfun.flags.used && pfun._module != null) {
+ if (function_index(program, pfun) >= 0 && is_used(program, pfun) && pfun._module != null) {
used |> insert(pfun._module)
}
})
})
- var utm = new UseTypeMarker()
+ var utm = new UseTypeMarker(program := program)
make_visitor(*utm) $(adapter) {
visit(program, adapter)
var foreign <- foreignUsedFunctions(program, false)
diff --git a/daslib/aot_standalone.das b/daslib/aot_standalone.das
index c24d064760..d1e4bfca6a 100644
--- a/daslib/aot_standalone.das
+++ b/daslib/aot_standalone.das
@@ -85,7 +85,7 @@ def collectInitFunctionAotNames(program : ProgramPtr; var context : Context) : a
let thisModule = program.getThisModule
program.get_ptr() |> for_each_module_no_order($(pm) {
pm |> for_each_module_function($(pfun) {
- if (pfun.index < 0 || !pfun.flags.used || !(initMnhSet |> key_exists(pfun.getMangledNameHash))) {
+ if (function_index(program, pfun) < 0 || !is_used(program, pfun) || !(initMnhSet |> key_exists(pfun.getMangledNameHash))) {
return
}
if (pm != thisModule) {
@@ -116,8 +116,8 @@ def private usedGlobals(var program : ProgramPtr) : array {
if (program.totalVariables > 0) {
program.get_ptr() |> for_each_module_no_order($(pm) {
pm |> for_each_global($(pvar) {
- if (!pvar.flags.used) return
- if (pvar.index < 0) {
+ if (!is_used(program, pvar)) return
+ if (variable_index(program, pvar) < 0) {
macro_error(program, pvar.at, "standalone AOT: global {pvar.name} is used but was never allocated an index")
return
}
@@ -136,8 +136,8 @@ def private standaloneCollects(program : ProgramPtr) : bool {
//! A `VarInfo` per global, made before the helper dumps its tables so every type they name is in
//! the dump: the collector walks each global through its info, the way the interpreter's
//! simulate step wires it.
-def private makeGlobalVarInfos(var helper : AotDebugInfoHelper?; globals : array) : array> {
- return <- [for (pvar in globals); (index = pvar.index, info = helper.helper |> make_variable_debug_info(pvar))]
+def private makeGlobalVarInfos(program : ProgramPtr; var helper : AotDebugInfoHelper?; globals : array) : array> {
+ return <- [for (pvar in globals); (index = variable_index(program, pvar), info = helper.helper |> make_variable_debug_info(pvar))]
}
def private writeGlobalVarInfos(var helper : AotDebugInfoHelper?; infos : array>; var tw : StringBuilderWriter) {
@@ -195,7 +195,7 @@ def private writeFunctionLookup(var tw : StringBuilderWriter; usedFunctions : ar
//! the standalone context's global lookup: the value is the offset InitGlobalVariable assigns at construction -
//! the 16-byte-rounded running size of the shared or the plain globals, in declaration order - and the
//! generated constructor verifies every global against it, so the two computations cannot drift apart silently
-def private writeVariableLookup(var tw : StringBuilderWriter; globals : array) {
+def private writeVariableLookup(program : ProgramPtr; var tw : StringBuilderWriter; globals : array) {
var lookup = unsafe(name_lookup_create())
var globalsSize = 0u
var sharedSize = 0u
@@ -207,7 +207,7 @@ def private writeVariableLookup(var tw : StringBuilderWriter; globals : array get_mangled_name()), string(pvar.name), uint(pvar.index), offset)
+ name_lookup_insert(lookup, hash(pvar |> get_mangled_name()), string(pvar.name), uint(variable_index(program, pvar)), offset)
}
name_lookup_seal(lookup)
writeLookupTables(tw, "var_lookup", lookup)
@@ -243,9 +243,10 @@ def writeStandaloneCtor(cfg : StandaloneContextCfg; initFunctions : string; var
write(tw, " // start totalVariables\n");
for (pvar in lookupVariableTable) {
- write(tw, " InitGlobalVar(context, &context.globalVariables[{pvar.index}/*pvar->index*/], GlobalVarInfo(\"")
+ let vindex = variable_index(program, pvar)
+ write(tw, " InitGlobalVar(context, &context.globalVariables[{vindex}/*pvar->index*/], GlobalVarInfo(\"")
write(tw, "{pvar.name}\", \"{pvar |> get_mangled_name()}\", {pvar._type.sizeOf}, {pvar.flags.global_shared}));\n");
- write(tw, " context.globalVariables[{pvar.index}].debugInfo = &__gvar_info_{pvar.index};\n");
+ write(tw, " context.globalVariables[{vindex}].debugInfo = &__gvar_info_{vindex};\n");
}
write(tw, " // end totalVariables\n\n");
write(tw, " context.allocateGlobalsAndShared();\n");
@@ -280,7 +281,7 @@ def writeStandaloneCtor(cfg : StandaloneContextCfg; initFunctions : string; var
write(tw, " \}\n");
}
- writeVariableLookup(tw, lookupVariableTable)
+ writeVariableLookup(program, tw, lookupVariableTable)
write(tw, " context.variableLookup = make_shared();\n");
write(tw, " context.variableLookup->adopt(var_lookup_table);\n");
write(tw, " for ( int i=0, is=context.totalVariables; i!=is; ++i ) \{\n");
@@ -340,7 +341,7 @@ class StandaloneContextGen : CppAot {
var globals : array;
prog.get_ptr() |> for_each_module($(pm) {
pm |> for_each_global($(pvar) {
- if (pvar.index < 0 || !pvar.flags.used
+ if (variable_index(prog, pvar) < 0 || !is_used(prog, pvar)
|| pvar._module == prog.getThisModule) return ;
globals.push(pvar);
});
@@ -493,7 +494,7 @@ def genStandaloneSrc(var program : ProgramPtr;
}
initFunctions = addFunctionInfo(collectProgramUsedFunctions(program, true, false), gen.helper);
- let globalInfos <- makeGlobalVarInfos(gen.helper, globals)
+ let globalInfos <- makeGlobalVarInfos(program, gen.helper, globals)
type_defs += gen.declarations;
gen.declarations = "";
write(tw, gen.str());
@@ -523,7 +524,7 @@ def private isMarkedNoAot(pfun : Function?) : bool {
def private checkAllUsedFunctionsCanAot(program : ProgramPtr) {
program.get_ptr() |> for_each_module_no_order($(pm) {
pm |> for_each_module_function($(pfun) {
- if (pfun.index < 0 || !pfun.flags.used || pfun.flags.builtIn) {
+ if (function_index(program, pfun) < 0 || !is_used(program, pfun) || pfun.flags.builtIn) {
return
}
if (pfun.flags.noAot) {
@@ -583,7 +584,7 @@ def private prepareProgramForEmission(var program : ProgramPtr; context : Contex
program.get_ptr() |> for_each_module_no_order($(pm) {
if (pm == thisModule) return
pm |> for_each_global($(pvar) {
- if (pvar.index < 0 || !pvar.flags.used || pvar.init == null) return
+ if (variable_index(program, pvar) < 0 || !is_used(program, pvar) || pvar.init == null) return
visit_expression(pvar.init, adapter_coll)
})
})
diff --git a/daslib/cross_context.das b/daslib/cross_context.das
new file mode 100644
index 0000000000..f6661d4819
--- /dev/null
+++ b/daslib/cross_context.das
@@ -0,0 +1,48 @@
+options gen2
+options indenting = 4
+options strict_smart_pointers
+
+module cross_context shared public
+
+//! A module required after the walk, and its macro context - `invoke_in_context` calls into it
+//! by a name held once there, under `unsafe`, the answer through a pointer.
+
+require daslib/ast public
+require daslib/rtti public
+require daslib/debugger public
+
+def private report_late_require(name : string; var mod : Module?; issues : das_string) : Module? {
+ if (mod == null) {
+ to_log(LOG_ERROR, "require_module_now({name}): {issues}")
+ }
+ return mod
+}
+
+def require_module_now(name : string) : Module? {
+ //! Compiles `name` - a `shared` module - and its prerequisites now, under the compiling
+ //! program's policies and file access, or answers the one already in the process; a panic
+ //! outside a compile (a macro, a simulate macro, an `[init]`). Null, logged, on failure.
+ if (!is_compiling()) {
+ panic("require_module_now({name}): no program is compiling - a require after the walk is issued from a macro, a simulate macro or an [init], not from a running script")
+ }
+ var found : Module?
+ var inscope no_access : smart_ptr
+ require_module_now(name, no_access, compiling_program().policies) $(var mod : Module?; issues : das_string) {
+ found = report_late_require(name, mod, issues)
+ }
+ return found
+}
+
+def macro_context_of(name : string) : Context? {
+ //! The macro context of the shared module `name`, required now when the process lacks
+ //! it; null, logged, when the module has no macros.
+ var mod = require_module_now(name)
+ if (mod == null) {
+ return null
+ }
+ if (!has_macro_context(mod)) {
+ to_log(LOG_ERROR, "macro_context_of({name}): the module has no macro context")
+ return null
+ }
+ return unsafe(addr(find_macro_context(mod)))
+}
diff --git a/daslib/just_in_time.das b/daslib/just_in_time.das
index 2750391988..6667db18ec 100644
--- a/daslib/just_in_time.das
+++ b/daslib/just_in_time.das
@@ -12,4 +12,4 @@ module just_in_time shared private
this is a JIT configuration file. We require platform appropriate jit module, as well as set necessary options.
*/
-require llvm/daslib/llvm_macro
+require llvm/daslib/llvm_jit_link
diff --git a/daslib/linq_fold.das b/daslib/linq_fold.das
index 13cfdd1bd8..fb4332f9ce 100644
--- a/daslib/linq_fold.das
+++ b/daslib/linq_fold.das
@@ -37,8 +37,9 @@ require daslib/linq_fold_array public
require daslib/linq_fold_decs public
require daslib/linq_fold_json public
require daslib/linq_fold_table public
-require ?pugixml pugixml/linq_fold_xml
require daslib/linq_fold_sql
+require daslib/module_group
+require [linq_fold_source]
[macro_function]
def private fold_linq_default(var expr : Expression?) : Expression? { // nolint:STYLE037 - one arm per linq call shape in the folded chain
@@ -125,6 +126,7 @@ def private register_all_linq_fold_rows {
build_group_by_rows()
build_join_rows()
build_zip_rows()
+ call_module_group("linq_fold_source", "register_linq_fold_source")
}
[macro_function]
@@ -180,15 +182,10 @@ def private try_splice_patterns(prog : ProgramPtr; var expr : Expression?) : Exp
return run_splice_adapter(calls, top, null,
new DecsAdapter(bridge = bridge, tupName = qn("decs_tup", at)), exprIsIter, at)
}
- static_if (typeinfo builtin_module_exists(pugixml)) {
- var xmlCall = extract_xml_source(top)
- if (xmlCall != null) {
- var topClone = clone_expression(top)
- var nameArg : Expression? = length(xmlCall.arguments) == 3 ? clone_expression(xmlCall.arguments[1]) : null
- return run_splice_adapter(calls, topClone, topClone,
- new XmlAdapter(nodeExpr = clone_expression(xmlCall.arguments[0]), srcName = qn("xnode", at),
- elemType = clone_type(xmlCall._type.firstType), nameArg = nameArg),
- exprIsIter, at)
+ for (recognize in linq_fold_sources) {
+ var m = invoke(recognize, top, at)
+ if (m.adapter != null) {
+ return run_splice_adapter(calls, m.matchTop, m.ctxTop, m.adapter, exprIsIter, at)
}
}
var jsonCall = extract_json_source(top)
diff --git a/daslib/linq_fold_common.das b/daslib/linq_fold_common.das
index 1d600b5be6..3dc721952f 100644
--- a/daslib/linq_fold_common.das
+++ b/daslib/linq_fold_common.das
@@ -290,6 +290,30 @@ struct EmitCtx {
typedef EmitFn = function<(var c : Captures; var ctx : EmitCtx; at : LineInfo) : Expression?>
+struct LinqFoldSourceMatch {
+ //! A source recognizer's answer: the adapter and the chain tops it runs under.
+ //! A null `adapter` is "not this source".
+ matchTop : Expression?
+ ctxTop : Expression?
+ adapter : SourceAdapter?
+}
+
+typedef LinqFoldSourceRecognizer = function<(var top : Expression?; at : LineInfo) : LinqFoldSourceMatch>
+
+var linq_fold_sources : array
+
+def add_linq_fold_source(recognizer : LinqFoldSourceRecognizer) {
+ //! Registers a source recognizer in the current macro context; `linq_fold` tries them
+ //! after its sql and decs sources and before json, table and array, in registration order.
+ //! A recognizer already registered is not added twice.
+ for (r in linq_fold_sources) {
+ if (r == recognizer) {
+ return
+ }
+ }
+ linq_fold_sources |> push(recognizer)
+}
+
struct SplicePattern {
name : string
chain : array
diff --git a/daslib/module_group.das b/daslib/module_group.das
new file mode 100644
index 0000000000..45ffc3c07a
--- /dev/null
+++ b/daslib/module_group.das
@@ -0,0 +1,67 @@
+options gen2
+options indenting = 4
+options strict_smart_pointers
+
+module module_group shared private
+
+//! Calls into every member of a module group.
+
+require daslib/ast_boost
+require daslib/templates_boost
+require daslib/fio
+require rtti
+require strings
+
+def public module_group_module_name(member : string) : string {
+ //! The module a group member's require path names: its last segment, less the `.das` a
+ //! relative spelling (`./foo.das`) may carry.
+ let name = base_name(member)
+ return extension(name) == ".das" ? slice(name, 0, length(name) - 4) : name
+}
+
+[call_macro(name="call_module_group")]
+class CallModuleGroup : AstCallMacro {
+ //! `call_module_group("group", "entry", args...)` expands to one `member::entry(args...)`
+ //! call per member, sorted by member path, an empty group to nothing; an argument that is
+ //! not a constant or a plain variable is bound to a local ahead of the calls, so it is evaluated once.
+ def override visit(prog : ProgramPtr; mod : Module?; expr : ExprCallMacro?) : ExpressionPtr {
+ if (length(expr.arguments) < 2 || !(expr.arguments[0] is ExprConstString) || !(expr.arguments[1] is ExprConstString)) {
+ macro_error(prog, expr.at, "call_module_group expects (\"group\", \"entry\", args...)")
+ return <- default
+ }
+ let group = string((expr.arguments[0] as ExprConstString).value)
+ let entry = string((expr.arguments[1] as ExprConstString).value)
+ var res <- qmacro_block() {
+ }
+ var blk = res as ExprBlock
+ var argNames : array
+ for (i in range(2, length(expr.arguments))) {
+ if (expr.arguments[i] is ExprVar || expr.arguments[i].__rtti |> starts_with("ExprConst")) {
+ argNames |> push("")
+ } else {
+ let name = make_unique_private_name("_cmg_arg_{i - 2}", expr.at) // hygienic: the block lands in the caller's scope
+ argNames |> push(name)
+ var decl <- qmacro_block() {
+ let $i(name) = $e(expr.arguments[i])
+ }
+ var declBlk = decl as ExprBlock
+ for (st in declBlk.list) {
+ blk.list |> emplace_new(clone_expression(st))
+ }
+ }
+ }
+ module_group_for_each_member(group) $(member : string) {
+ var call <- make_call(expr.at, "{module_group_module_name(member)}::{entry}")
+ for (i in range(2, length(expr.arguments))) {
+ let name = argNames[i - 2]
+ if (empty(name)) {
+ (call as ExprCall).arguments |> emplace_new(clone_expression(expr.arguments[i]))
+ } else {
+ (call as ExprCall).arguments |> emplace_new(qmacro($i(name)))
+ }
+ }
+ blk.list |> emplace_new(call)
+ }
+ return <- res
+ }
+}
diff --git a/daslib/sql_boost.das b/daslib/sql_boost.das
index 1f3eb3e23d..99d60ca56f 100644
--- a/daslib/sql_boost.das
+++ b/daslib/sql_boost.das
@@ -22,10 +22,9 @@ require daslib/templates_boost
require daslib/json public
require daslib/json_boost public
require daslib/archive public
+require daslib/module_group
-require ?sqlite sqlite/sqlite_provider
-require ?duckdb duckdb/duckdb_provider
-require ?postgres postgres/postgres_provider
+require [sql_provider]
//! Provider-neutral SQL macro layer: ``[sql_table]`` / ``[sql_view]`` / ``[sql_index]``,
//! the typed ALTER macros (``add_column`` / ``create_index``), and the ``sql_bind`` /
@@ -35,19 +34,11 @@ require ?postgres postgres/postgres_provider
//! (e.g. ``sqlite/sqlite_boost``). Contract: modules/dasSQLITE/PROVIDER_CONTRACT.md.
def public sql_register_present_providers {
- //! Registers every linked provider, once per macro context; call from a ``[_macro]``
- //! in each registry-consulting macro module (``[init]`` does not reach macro contexts).
- //! Adding a provider = one ``require ?`` line above + one ``static_if`` branch here.
+ //! Registers every present provider, once per macro context; call from a ``[_macro]`` in
+ //! each registry-consulting macro module (``[init]`` does not reach macro contexts). A
+ //! provider joins the ``sql_provider`` group from its descriptor and defines ``register_provider``.
if (sql_register_present_providers_done()) return
- static_if (typeinfo builtin_module_exists(sqlite)) {
- register_sqlite_provider()
- }
- static_if (typeinfo builtin_module_exists(duckdb)) {
- register_duckdb_provider()
- }
- static_if (typeinfo builtin_module_exists(postgres)) {
- register_postgres_provider()
- }
+ call_module_group("sql_provider", "register_provider")
}
[_macro]
diff --git a/daslib/sql_provider.das b/daslib/sql_provider.das
index 75cb301a2f..a21700763b 100644
--- a/daslib/sql_provider.das
+++ b/daslib/sql_provider.das
@@ -14,9 +14,9 @@ require strings
//! everything dialect- or handle-specific routes through a `SqlProviderInfo` entry
//! registered here. Registries live per macro context (see skills/das_macros.md "Macro
//! modules each compile into their own context"), so each consulting macro module calls
-//! `sql_register_present_providers` (in `daslib/sql_boost`, which pulls each present
-//! provider's shim in via `require ?` — this core module stays provider-blind so
-//! shims can require it without cycling). Contract: modules/dasSQLITE/PROVIDER_CONTRACT.md.
+//! `sql_register_present_providers` (in `daslib/sql_boost`, which pulls every shim in
+//! through the `sql_provider` group — this core module stays provider-blind so shims
+//! can require it without cycling). Contract: modules/dasSQLITE/PROVIDER_CONTRACT.md.
enum public SqlPkReport {
//! How `insert(row)` learns the new row's primary key.
diff --git a/daslib/style_lint.das b/daslib/style_lint.das
index 35c0193559..a826b14d49 100644
--- a/daslib/style_lint.das
+++ b/daslib/style_lint.das
@@ -35,8 +35,8 @@ module style_lint shared private
//! STYLE026 — nested 'unsafe { ... }' block; outer wrap already covers the scope — drop the inner
//! STYLE027 — var array / table with empty default-init followed by a for-loop that only push/insert into it; use a comprehension
//! STYLE028 — 'self->method(...)' inside a class method is optional — drop 'self->' and call 'method(...)' directly (compiler auto-promotes to the same invoke)
-//! STYLE029 — non-public 'require X' used only for ONE module X re-exports — require it directly and drop X (skipped when ≥2 re-exports are used: aggregation facade; or when X provides macros or [init])
-//! STYLE030 — non-public 'require X' that is entirely unused — no symbol from X (or anything it re-exports) is referenced; drop it (skipped when X provides any macro, or an [init]/[finalize]/[before_reload]/[after_reload]/[live_command] function, or only re-exports builtins used through it, or X or its public re-export closure exports a function matching an unresolved call inside an uninstanced generic body). Suppress a deliberate keep with '// nolint:STYLE030'
+//! STYLE029 — non-public 'require X' used only for ONE module X re-exports — require it directly and drop X (skipped when ≥2 re-exports are used: aggregation facade; when X provides macros or [init]; or when X is a member of a 'require [group]', whose members are one unit)
+//! STYLE030 — non-public 'require X' that is entirely unused — no symbol from X (or anything it re-exports) is referenced; drop it (skipped when X provides any macro, or an [init]/[finalize]/[before_reload]/[after_reload]/[live_command] function, or only re-exports builtins used through it, or X or its public re-export closure exports a function matching an unresolved call inside an uninstanced generic body). A 'require [group]' is one unit: reported only when no member is used. Suppress a deliberate keep with '// nolint:STYLE030'
//! STYLE031 — table (or table set) var initialized via a run of insert / '[]=' statements; use a table (set) literal move-assign
//! STYLE032 — array with empty default-init then a single 'push_from' / 'push_clone_from' from another array; that is a clone — use 'var w := SRC' (clone-assign). Cloneable elements only (a copyable-but-not-cloneable element such as a lambda stays silent — ':=' would not compile)
//! STYLE033 — run of >=2 'push_from' / 'push_clone_from' from arrays; into a fresh-empty array that is a concatenation (use 'var c <- concat(s1, s2, ...)', needs 'require daslib/linq'; cloneable elements only, as STYLE032); into an existing array — plain var or a pure chain like 'obj.arr' / 'bags[0].nums' — collapse to one variadic call of the same name ('dst |> push_from(...)' or 'dst |> push_clone_from(...)')
@@ -2264,15 +2264,35 @@ class StyleLintVisitor : AstVisitor {
}
}
}
+ st030_report_requires(prog, module_files, this_name, hook_reach)
+ }
+
+ [arch(at="ARCHITECTURE_LINT.md#style-lint-require-group")]
+ def st030_report_requires(prog : ProgramPtr; module_files : table; this_name : string; hook_reach : table) : void {
+ var group_line : table
+ prog |> for_each_require_declaration() $ [unused_argument(req_name, file)] (mod, req_name, file, pub, at) {
+ if (mod == null || pub || at.fileInfo == null || !key_exists(module_files, string(at.fileInfo.name))) return
+ let key = st030_decl_key(at)
+ if (!key_exists(group_line, key)) {
+ group_line[key] = st030_is_group_line(rtti_get_source_line(at.fileInfo, at.line))
+ }
+ }
+ var keep : table
+ var verdict : table
+ var members : table
prog |> for_each_require_declaration() $ [unused_argument(file)] (mod, req_name, file, pub, at) {
if (mod == null || pub
|| at.fileInfo == null || !key_exists(module_files, string(at.fileInfo.name))) return
+ let key = st030_decl_key(at)
let mname = "{mod.name}"
if (mname == "" || mname == this_name || mname == "$" || mname == "builtin"
|| key_exists(used_modules, mname)
|| st029_module_provides_macro(mod)
|| st029_module_has_lifecycle_hook(mod)
- || st030_exports_unresolved_name(mod)) return
+ || st030_exports_unresolved_name(mod)) {
+ keep[key] = true
+ return
+ }
var closure : table
st029_collect_public_deps(mod, closure)
var via : array
@@ -2282,19 +2302,77 @@ class StyleLintVisitor : AstVisitor {
used_count++
if (closure?[cm] ?? false) via |> push(cm)
}
- var line_at = at
var reached : table
reached |> insert(mname, mod)
st029_collect_all_deps(mod, reached)
if (used_count == 1 && !empty(via)) {
let mods = via[0]
- return if (st029_hook_only_through(reached, hook_reach, reached?[mods] ?? null))
- style_warning("STYLE029: require {req_name} is used only for the module it re-exports ({mods}); require it directly and drop {req_name}", line_at)
+ if ((group_line?[key] ?? false) || st029_hook_only_through(reached, hook_reach, reached?[mods] ?? null)) {
+ keep[key] = true
+ return
+ }
+ verdict["{key}|{req_name}"] = "029:{mods}"
+ return
+ }
+ if (used_count > 0 || st029_hook_only_through(reached, hook_reach, null)) {
+ keep[key] = true
+ return
+ }
+ verdict["{key}|{req_name}"] = "030"
+ let listed = members?[key] ?? ""
+ members[key] = listed == "" ? "{req_name}" : "{listed}, {req_name}"
+ }
+ var group_reported : table
+ prog |> for_each_require_declaration() $ [unused_argument(mod, file, pub)] (mod, req_name, file, pub, at) {
+ if (at.fileInfo == null || !key_exists(module_files, string(at.fileInfo.name))) return
+ let key = st030_decl_key(at)
+ if (keep?[key] ?? false) return
+ let v = verdict?["{key}|{req_name}"] ?? ""
+ if (v == "") return
+ if (starts_with(v, "029:")) {
+ let mods = slice(v, 4)
+ style_warning("STYLE029: require {req_name} is used only for the module it re-exports ({mods}); require it directly and drop {req_name}", at)
+ return
+ }
+ if (group_line?[key] ?? false) {
+ if (group_reported?[key] ?? false) return
+ group_reported[key] = true
+ style_warning("STYLE030: require [group] is unused - no symbol from any member ({members?[key] ?? ""}) or a module it re-exports is referenced; drop it", at)
return
}
- return if (used_count > 0 || st029_hook_only_through(reached, hook_reach, null))
- style_warning("STYLE030: require {req_name} is unused - no symbol from it (or a module it re-exports) is referenced; drop it", line_at)
+ style_warning("STYLE030: require {req_name} is unused - no symbol from it (or a module it re-exports) is referenced; drop it", at)
+ }
+ }
+
+ def st030_decl_key(at : LineInfo) : string {
+ return "{at.fileInfo.name}:{at.line}:{at.column}"
+ }
+
+ def st030_is_group_line(src : string) : bool {
+ //! `require [group]` or `require ?guard [group]` - the bracket right after the keyword
+ //! and the optional guard, not one anywhere on the line
+ var group = false
+ peek_data(src) $(d) {
+ let req = find(d, "require")
+ if (req < 0) {
+ return
+ }
+ var i = req + length("require")
+ let n = length(d)
+ while (i < n && is_tab_or_space(int(d[i]))) {
+ i++
+ }
+ if (i < n && int(d[i]) == '?') {
+ while (i < n && !is_tab_or_space(int(d[i])) && int(d[i]) != '[') {
+ i++
+ }
+ while (i < n && is_tab_or_space(int(d[i]))) {
+ i++
+ }
+ }
+ group = i < n && int(d[i]) == '['
}
+ return group
}
def note_decl_span(var file_first_decl : table; fname : string; s : uint) : void {
diff --git a/daslib/tune.das b/daslib/tune.das
index ec3f2fb079..1ae4264b2d 100644
--- a/daslib/tune.das
+++ b/daslib/tune.das
@@ -7,17 +7,16 @@ module tune shared public
require daslib/ast
require daslib/ast_boost
-require ?llvm llvm/daslib/llvm_code public
-require ?llvm llvm/daslib/llvm_tune public
+require [tune_framework] public
//! The six kernel-tuning annotations - `[tune_perm]`, `[tune_companion]`, `[tune]`,
//! `[tune_scope]`, `[tune_policy]` and `[llvm_code]` - plus the default-policy infer pass, as
//! shells every program can spell. The framework that implements them lives in
-//! `llvm/daslib/llvm_tune` and `llvm/daslib/llvm_code`, reached through the `?llvm` guards
-//! above and re-exported, so `require daslib/tune` gives exactly what `require llvm/daslib/llvm_tune`
-//! gave. The guard is the `llvm` C++ witness module, compiled in exactly when the build is
-//! configured with dasLLVM - the das files being on disk never counts; the guarded require is
-//! then what makes `module_exists` below answer for this compile (`module_exists`, not
+//! `llvm/daslib/llvm_tune` - the `tune_framework` group's member, re-exported, and itself
+//! re-exporting `llvm/daslib/llvm_code` - so `require daslib/tune` gives exactly what
+//! `require llvm/daslib/llvm_tune` gave. dasLLVM joins the group exactly when the build is
+//! configured with it - from its `llvm` witness module and its descriptor - the das files being on
+//! disk never counts; the member is then what makes `module_exists` below answer for this compile (`module_exists`, not
//! `builtin_module_exists`: visible from this module, the same answer inside a tool's nested
//! compile). Every tune annotation registers HERE and nowhere else, because a
//! module's macro state is per-module: the `[tune_scope]` bank and the pass that reads it have
diff --git a/daslib/validate_code.das b/daslib/validate_code.das
index fe28f4ea67..d362012c6a 100644
--- a/daslib/validate_code.das
+++ b/daslib/validate_code.das
@@ -141,12 +141,12 @@ class JIT_LLVM : AstSimulateMacro {
make_visitor(*astVisitor) $(adapter) {
prog |> for_each_module() $(mod) {
mod |> for_each_function("") $(f) {
- if (f.flags.used) {
+ if (is_used(prog, f)) {
visit(f, adapter)
}
}
mod |> for_each_global() $(g) {
- if (g.flags.used) {
+ if (is_used(prog, g)) {
if (!g._type.isNoHeapType) {
macro_error(compiling_program(), g.at, "global variable {g.name} requires heap")
} elif (g.init != null) {
diff --git a/dastest/dastest.das b/dastest/dastest.das
index 1be2cf0ff5..44bab67979 100644
--- a/dastest/dastest.das
+++ b/dastest/dastest.das
@@ -436,9 +436,12 @@ def deserialize_path(var ctx : SuiteCtx, _files : array, in_file : strin
fread(f, data)
log::info("Deserializing {count:d} programs from {in_file}")
var ser = create_ast_deserializer(data)
+ // the reader's access: a stream carries none, and a program's late require takes its program's
+ var inscope access <- make_file_access(ctx.projectPath)
+ access |> add_file_access_root("dastest", ctx.dastestRoot)
using() $(var _mg : ModuleGroup) {
for (i in range(count)) {
- deserialize_program(ser) $(ok, program, error) {
+ deserialize_program(ser, access) $(ok, program, error) {
if (!ok) {
log::error("Deserialization failed at program {i}: {error}")
res.errors++
@@ -755,7 +758,7 @@ def main() : int { // nolint:STYLE037,STYLE038 - CLI dispatch, one arm per mode
log::info("to get more information, run the test manually\n> {isoRes.cmd}")
} elif (!isoRes.parsedResult) {
reason = "crashed (exit status {isoRes.exitCode})"
- log::red("`{isoRes.uri}` execution was interrupted, crashed for example")
+ log::red("`{isoRes.uri}` ended without a result: the child exited with status {isoRes.exitCode} before printing its summary - a crash, an abort, or a panic outside any test")
log::info("to get more information, run the test manually\n> {isoRes.cmd}")
} else {
reason = "{isoRes.status.failed} failed, {isoRes.status.errors} errors"
diff --git a/dastest/review_gate.das b/dastest/review_gate.das
index ad1fe6437f..0640fde1cc 100644
--- a/dastest/review_gate.das
+++ b/dastest/review_gate.das
@@ -199,7 +199,8 @@ def private quoted_ident_tokens(text : string) : array {
//! Descriptor census for a module whose CMake derives its resolver rows from `.das_module`
//! (ADD_MODULE_DAS_FROM_DESCRIPTOR), both directions: every `.das` in `census_dirs` is
//! named by a quoted token in the descriptor, and every identifier-shaped quoted token
-//! outside `non_module_tokens` (e.g. the require category, a DLL registration class)
+//! outside `non_module_tokens` (e.g. the require category, a DLL registration class) and
+//! outside a `register_module_group` row (a registry key and a require path, never a file)
//! names a file in some census dir. Neither direction has a build-time signal - a gate
//! built on this is the only enforcer.
def gate_descriptor_census(descriptor_path : string; census_dirs : array; non_module_tokens : table) {
@@ -207,7 +208,7 @@ def gate_descriptor_census(descriptor_path : string; census_dirs : array
gate_finding(descriptor_path, "missing - the descriptor is the module's one registration surface")
return
}
- let descriptor = strip_line_comments(fread(descriptor_path))
+ let descriptor = without_group_rows(strip_line_comments(fread(descriptor_path)))
for (folder in census_dirs) {
dir(folder) $(name) {
if (name |> ends_with(".das")) {
@@ -229,6 +230,10 @@ def gate_descriptor_census(descriptor_path : string; census_dirs : array
}
}
+def private without_group_rows(text : string) : string {
+ return join([for (line in split(text, "\n")); line; where find(line, "register_module_group(") < 0], "\n")
+}
+
def private any_dir_holds(census_dirs : array; fname : string) : bool {
for (folder in census_dirs) {
if (fexist("{folder}/{fname}")) {
diff --git a/doc/internal/error_audit.csv b/doc/internal/error_audit.csv
index 251adcc822..073d82c35d 100644
--- a/doc/internal/error_audit.csv
+++ b/doc/internal/error_audit.csv
@@ -729,7 +729,7 @@ src/ast/ast_lint.cpp,392,cant_be_null,global variable of type '' can't be i
src/ast/ast_lint.cpp,397,invalid_variable_type,global variable '' is too big,LintVisitor::preVisitGlobalLet,primary,exceeds | global,exceeds_global,
src/ast/ast_lint.cpp,417,variable_not_found,global variable initialization loop,LintVisitor::preVisitGlobalLetInit,recategorize,recursion | global | loop,recursion_global_loop,"recategorize: variable_not_found is wrong; this is initialization-recursion"
src/ast/ast_lint.cpp,425,no_init,[init] is disabled in the options or CodeOfPolicies,LintVisitor::visitGlobalLetInit,primary,cant | global,cant_global,
-src/ast/ast_lint.cpp,437,variable_not_found,global variable is initialized after (),LintVisitor::preVisit(ExprVar*),recategorize,invalid | global,invalid_global,"recategorize: variable_not_found is wrong; this is initialization-order"
+src/ast/ast_lint.cpp,437,variable_not_found,global variable is initialized after ,LintVisitor::preVisit(ExprVar*),recategorize,invalid | global,invalid_global,"recategorize: variable_not_found is wrong; this is initialization-order"
src/ast/ast_lint.cpp,451,invalid_name,invalid variable name '',LintVisitor::preVisit(ExprFor*),primary,invalid | variable | name,invalid_variable_name,
src/ast/ast_lint.cpp,459,cant_be_null,can't delete it will create null pointer,LintVisitor::preVisit(ExprDelete*),primary,cant | expression,cant_expression,
src/ast/ast_lint.cpp,470,invalid_name,invalid variable name ,LintVisitor::preVisit(ExprLet*),primary,invalid | variable | name,invalid_variable_name,
diff --git a/doc/reflections/das2rst.das b/doc/reflections/das2rst.das
index 18bed235e5..27da3f2a95 100644
--- a/doc/reflections/das2rst.das
+++ b/doc/reflections/das2rst.das
@@ -32,6 +32,8 @@ require daslib/tty
require daslib/jobque_boost
require daslib/jobque_profile
require daslib/apply_in_context
+require daslib/cross_context
+require daslib/module_group
require daslib/contracts
require daslib/defer
require daslib/with_boost
@@ -146,7 +148,7 @@ require strudel/strudel_sf2_voice
require strudel/strudel_pattern
require spirv/spirv_reflect
require dasllama/dasllama
-require ?llvm llvm/daslib/llvm_tune // nolint:STYLE030,LINT019 — the framework's is_building_documentation check keeps [tune] inert; a build without dasLLVM has no framework to keep inert
+require [tune_framework] // nolint:STYLE030,LINT019 — the framework's is_building_documentation check keeps [tune] inert; a build without dasLLVM has no framework to keep inert
def document_module_math(_root : string) {
var mod = get_module("math")
@@ -230,7 +232,7 @@ def document_module_fio(_root : string) {
group_by_regex("Filesystem queries", mod, %regex~(temp_directory|temp_directory_result|create_temp_file|create_temp_file_result|create_temp_directory|create_temp_directory_result|disk_space)$%%),
group_by_regex("Terminal queries", mod, %regex~(is_terminal|terminal_width)$%%),
group_by_regex("OS specific routines", mod, %regex~(sleep|exit|system|popen|popen_binary|popen_timeout|spawn_argv|popen_argv|popen_argv_pipe|popen_timed_out|run_and_capture|spawn_process|process_drain|process_poll|process_wait|process_terminate|process_kill|process_pid|process_alive|close_process|process_running|with_process|get_env_variable|set_env_variable|sanitize_command_line|has_env_variable)$%%),
- group_by_regex("Dynamic modules", mod, %regex~(register_dynamic_module|register_native_path|describe_pending_dynamic_modules|no_manifest)$%%)
+ group_by_regex("Dynamic modules", mod, %regex~(register_dynamic_module|register_native_path|register_module_group|describe_pending_dynamic_modules|no_manifest)$%%)
)
documents("File input output library", mod, "fio.rst", groups)
}
@@ -445,7 +447,7 @@ def document_module_rtti(_root : string) {
group_by_regex("Initialization and finalization", mod, %regex~(using|LineInfo|CodeOfPolicies|RttiValue_nothing)$%%),
group_by_regex("Type access", mod, %regex~(get_dim|builtin_is_same_type|is_compatible_cast|get_das_type_name|is_same_type|each_dim|arg_types|arg_names|get_type_size|get_type_align)$%%),
group_by_regex("Rtti context access", mod, %regex~(get_total_functions|get_total_variables|get_function_info|get_variable_info|get_variable_value|get_function_by_mnh|get_line_info|this_context|context_for_each_function|context_for_each_variable|for_each_init_function|class_info|type_info)$%%),
- group_by_regex("Program access", mod, %regex~(program_for_each_module|program_for_each_registered_module|get_this_module|get_module|has_module)$%%),
+ group_by_regex("Program access", mod, %regex~(program_for_each_module|program_for_each_registered_module|get_this_module|get_module|has_module|module_group_for_each_member)$%%),
group_by_regex("Module access", mod, %regex~(module_for_each_structure|module_for_each_enumeration|module_for_each_function|module_for_each_generic|module_for_each_global|module_for_each_annotation|module_for_each_dependency)$%%),
group_by_regex("Annotation access", mod, %regex~(get_annotation_argument_value|add_annotation_argument|get_annotation|get_annotation_argument|resolve_annotation|each_annotation|each_annotation_argument)$%%),
group_by_regex("Compilation and simulation", mod, %regex~(compile|compile_file|for_each_expected_error|for_each_require_declaration|simulate|create_ast_serializer|create_ast_deserializer|delete_ast_serializer|serialize_program|deserialize_program|ast_serializer_get_data)$%%),
@@ -467,14 +469,14 @@ def document_module_rtti(_root : string) {
def document_module_ast(_root : string) {
var mod = [get_module("ast_core"), find_module("ast")]
var groups <- array(
- group_by_regex("Compilation and file access", mod, %regex~(compile|compile_file|parse_file|make_file_access)$%%),
+ group_by_regex("Compilation and file access", mod, %regex~(compile|compile_file|parse_file|make_file_access|require_module_now)$%%),
group_by_regex("Call generation", mod, %regex~(make_call)$%%),
group_by_regex("Visitor pattern", mod, %regex~visit.*%%),
group_by_regex("Expression generation", mod, %regex~(force_generated|get_expression_annotation|make_type_info_structure|stamp_missing_at)%%),
group_by_regex("Adapter generation", mod, %regex~make_.*%%),
group_by_regex("Adapter application", mod, %regex~(add_.*_(annotation|macro)|add_new_comment_reader|add_comment_reader|add_module_option)%%),
group_by_regex("Adding objects to objects", mod, %regex~add_.*%%),
- group_by_regex("Program and module access", mod, %regex~(this_program|this_module|compiling_program|compiling_module)$%%),
+ group_by_regex("Program and module access", mod, %regex~(this_program|this_module|compiling_program|compiling_module|has_macro_context|find_macro_context|is_used|function_index|variable_index|is_jit_selected|set_jit_selected|clear_jit_selection)$%%),
group_by_regex("Textual descriptions of the objects", mod, %regex~(describe.*|das_to_string)%%),
group_by_regex("Searching", mod, %regex~find.*%%),
group_by_regex("Iterating", mod, %regex~(for_each.*|.*_foreach)%%),
@@ -699,6 +701,23 @@ def document_module_apply_in_context(_root : string) {
document("Cross-context evaluation helpers", mod, "apply_in_context.rst", groups)
}
+def document_module_cross_context(_root : string) {
+ var mod = find_module("cross_context")
+ var groups <- array(
+ group_by_regex("Late require", mod, %regex~(require_module_now|macro_context_of)$%%),
+ group_by_regex("Calls into a macro context", mod, %regex~(call_in_context)$%%)
+ )
+ document("A module required after the walk, and calls into its macro context", mod, "cross_context.rst", groups)
+}
+
+def document_module_module_group(_root : string) {
+ var mod = find_module("module_group")
+ var groups <- array(
+ group_by_regex("Group members", mod, %regex~(module_group_module_name)$%%)
+ )
+ document("Calls into every member of a module group", mod, "module_group.rst", groups)
+}
+
def document_module_ansi_colors(_root : string) {
var mod = find_module("ansi_colors")
var groups <- array(
@@ -1950,6 +1969,8 @@ def main { // nolint:STYLE038 — flat one-call-per-documented-module dispatch
document_module_ansi_colors(root)
document_module_tty(root)
document_module_apply_in_context(root)
+ document_module_cross_context(root)
+ document_module_module_group(root)
document_module_apply(root)
document_module_archive(root)
document_module_array_boost(root)
diff --git a/doc/source/reference/language/lint.rst b/doc/source/reference/language/lint.rst
index 9a06ffa78b..cdd3e4bdc3 100644
--- a/doc/source/reference/language/lint.rst
+++ b/doc/source/reference/language/lint.rst
@@ -2853,7 +2853,9 @@ A non-public ``require X`` where no symbol from ``X`` (or any module it
re-exports) is referenced anywhere in the file. Remove it. Skipped when ``X``
provides any macro or an ``[init]``, when a lifecycle hook is reachable only
through ``X`` (as for STYLE029), or when ``X`` only re-exports builtins used
-through it. Suppress a deliberate keep with ``// nolint:STYLE030``.
+through it. A ``require [group]`` is one unit: it is reported only when no
+member is used, and a member used through its re-export never draws STYLE029.
+Suppress a deliberate keep with ``// nolint:STYLE030``.
.. das-doc: alt
.. code-block:: das
diff --git a/doc/source/reference/language/modules.rst b/doc/source/reference/language/modules.rst
index d3d4335a4c..aeb72078a9 100644
--- a/doc/source/reference/language/modules.rst
+++ b/doc/source/reference/language/modules.rst
@@ -123,6 +123,52 @@ branch before name resolution, so the symbols are referenced only when present:
parse_xml("") $(doc, ok) { /* ... */ }
}
+-------------
+Module groups
+-------------
+
+A ``require`` may name a **group** in square brackets instead of a module (gen2
+syntax only):
+
+.. code-block:: das
+
+ require [sql_provider]
+ require ?sqlite [sql_provider] public
+
+A group is a name that modules register themselves under. A module's ``.das_module``
+descriptor calls ``register_module_group("sql_provider", "sqlite/sqlite_provider", "sqlite")``
+- the third argument, optional, is the member's guard, what ``require ?sqlite
+sqlite/sqlite_provider`` would carry, so a member whose file requires a C++ module the
+build may lack joins only where that module is - and a C++ module registers with
+``registerModuleGroupMember``. ``require [group]`` is one ordinary ``require`` per
+registered member, sorted by member path: a member resolves and fails exactly as the same
+require spelled by hand would, ``public`` applies to every member, a guard on the group
+drops the whole group the way it drops a single require, and a member's own guard drops
+that member. A group nothing registered under adds nothing.
+
+The group turns the dependency around. A module that wants every installed provider
+would otherwise have to name each one with its own guard and gain a line per
+provider; with a group each provider names the group it joins, and the requirer
+names only the group. The module cache records the requires a parse took, so a
+member joining a group later re-parses the modules that require the group.
+
+The requirer reaches its members without naming them through ``daslib/module_group``:
+
+.. code-block:: das
+
+ require daslib/module_group
+ require [sql_provider]
+
+ def register_present_providers {
+ call_module_group("sql_provider", "register_provider")
+ }
+
+``call_module_group("group", "entry", args...)`` expands at compile time to one
+``member::entry(args...)`` call per registered member, in the group's sorted order; a member
+without the entry is a compile error naming the call, and an empty group expands to
+nothing. A group is therefore two names: the group a member joins, and the entry every
+member defines.
+
--------------
Native modules
--------------
diff --git a/doc/source/stdlib/handmade/function-ast-clear_jit_selection-0xd4c94e0272357bbb.rst b/doc/source/stdlib/handmade/function-ast-clear_jit_selection-0xd4c94e0272357bbb.rst
new file mode 100644
index 0000000000..b170d5b43a
--- /dev/null
+++ b/doc/source/stdlib/handmade/function-ast-clear_jit_selection-0xd4c94e0272357bbb.rst
@@ -0,0 +1,2 @@
+Empties ``program``'s JIT selection - the state ``mark_jit_selection`` rebuilds before it
+marks the functions a DLL holds. A null ``program`` throws.
diff --git a/doc/source/stdlib/handmade/function-ast-clear_jit_selection-0xe0b782a1c6f1586c.rst b/doc/source/stdlib/handmade/function-ast-clear_jit_selection-0xe0b782a1c6f1586c.rst
new file mode 100644
index 0000000000..e936d25ad5
--- /dev/null
+++ b/doc/source/stdlib/handmade/function-ast-clear_jit_selection-0xe0b782a1c6f1586c.rst
@@ -0,0 +1 @@
+The same for a ``ProgramPtr``.
diff --git a/doc/source/stdlib/handmade/function-ast-find_macro_context-0x598d596c1e881f4f.rst b/doc/source/stdlib/handmade/function-ast-find_macro_context-0x598d596c1e881f4f.rst
new file mode 100644
index 0000000000..57d9959f2f
--- /dev/null
+++ b/doc/source/stdlib/handmade/function-ast-find_macro_context-0x598d596c1e881f4f.rst
@@ -0,0 +1 @@
+Returns the macro context of ``module`` - every macro context carries the mutex ``invoke_in_context`` locks from its creation. Throws when the module has no macro context - ask ``has_macro_context`` first.
diff --git a/doc/source/stdlib/handmade/function-ast-function_index-0x8093f578250f3115.rst b/doc/source/stdlib/handmade/function-ast-function_index-0x8093f578250f3115.rst
new file mode 100644
index 0000000000..7418b6284d
--- /dev/null
+++ b/doc/source/stdlib/handmade/function-ast-function_index-0x8093f578250f3115.rst
@@ -0,0 +1,4 @@
+The slot ``function`` holds in ``program``'s context: ``-1`` when the program never
+allocated it, ``-2`` when the allocation found it unused, its function-table index
+otherwise. The index is the program's, not the function's - a shared module's function
+has one per program that requires it.
diff --git a/doc/source/stdlib/handmade/function-ast-has_macro_context-0x4f257a25fcd331a4.rst b/doc/source/stdlib/handmade/function-ast-has_macro_context-0x4f257a25fcd331a4.rst
new file mode 100644
index 0000000000..ee1b4007dc
--- /dev/null
+++ b/doc/source/stdlib/handmade/function-ast-has_macro_context-0x4f257a25fcd331a4.rst
@@ -0,0 +1 @@
+Returns true when ``module`` has a macro context - it declares a macro (a ``[_macro]`` function is the smallest) and was compiled with one. A null module answers false.
diff --git a/doc/source/stdlib/handmade/function-ast-is_jit_selected-0x3081601f8756204d.rst b/doc/source/stdlib/handmade/function-ast-is_jit_selected-0x3081601f8756204d.rst
new file mode 100644
index 0000000000..3f868d2ba1
--- /dev/null
+++ b/doc/source/stdlib/handmade/function-ast-is_jit_selected-0x3081601f8756204d.rst
@@ -0,0 +1,6 @@
+True when ``program``'s JIT selected ``function`` - the DLL the program runs holds it, so a
+call to it from jitted code goes into the DLL rather than through an interpreter slot. The
+selection is the program's, not the function's: a shared module's function is selected by
+one program and not by another in the same process. ``[jit]`` on the function is the
+author's request (``func.moreFlags.requestJit``), which the plan reads and the selection
+never writes. A null ``program`` throws.
diff --git a/doc/source/stdlib/handmade/function-ast-is_used-0x5ff424eace25cf4c.rst b/doc/source/stdlib/handmade/function-ast-is_used-0x5ff424eace25cf4c.rst
new file mode 100644
index 0000000000..ba980a1876
--- /dev/null
+++ b/doc/source/stdlib/handmade/function-ast-is_used-0x5ff424eace25cf4c.rst
@@ -0,0 +1,4 @@
+True when ``program``'s symbol-use pass reached ``function``. The answer is the
+program's, not the function's: a shared module's function is used by one program and not
+by another compiled in the same process, and a compile nested inside this one leaves the
+answer standing.
diff --git a/doc/source/stdlib/handmade/function-ast-require_module_now-0x3321f96caa45428c.rst b/doc/source/stdlib/handmade/function-ast-require_module_now-0x3321f96caa45428c.rst
new file mode 100644
index 0000000000..570594634f
--- /dev/null
+++ b/doc/source/stdlib/handmade/function-ast-require_module_now-0x3321f96caa45428c.rst
@@ -0,0 +1 @@
+Compiles ``module_name`` - a ``shared`` module, named by its file - and its prerequisites into the process at the point of the call, or answers the module already there, under ``codeOfPolicies``; a null ``fileAccess`` means the compiling program's own. Issued from a compile - a macro, a simulate macro, an ``[init]`` - and an error from a running script, which has no compile for the modules to join. Calls ``block`` with the module (null on failure) and the compile log. The module is not a dependency of the caller: reach it through its macro context (``find_macro_context``, ``invoke_in_context``).
diff --git a/doc/source/stdlib/handmade/function-ast-set_jit_selected-0xb4bb112e5f39b93c.rst b/doc/source/stdlib/handmade/function-ast-set_jit_selected-0xb4bb112e5f39b93c.rst
new file mode 100644
index 0000000000..e936d25ad5
--- /dev/null
+++ b/doc/source/stdlib/handmade/function-ast-set_jit_selected-0xb4bb112e5f39b93c.rst
@@ -0,0 +1 @@
+The same for a ``ProgramPtr``.
diff --git a/doc/source/stdlib/handmade/function-ast-set_jit_selected-0xe95e6598b9f2772b.rst b/doc/source/stdlib/handmade/function-ast-set_jit_selected-0xe95e6598b9f2772b.rst
new file mode 100644
index 0000000000..af4de5d07d
--- /dev/null
+++ b/doc/source/stdlib/handmade/function-ast-set_jit_selected-0xe95e6598b9f2772b.rst
@@ -0,0 +1,2 @@
+Marks ``function`` as selected by ``program``'s JIT (``selected`` true) or drops it from the
+selection (false). A null ``program`` throws; a null ``function`` is ignored.
diff --git a/doc/source/stdlib/handmade/function-ast-variable_index-0xbf5517480aaeffb1.rst b/doc/source/stdlib/handmade/function-ast-variable_index-0xbf5517480aaeffb1.rst
new file mode 100644
index 0000000000..936488283e
--- /dev/null
+++ b/doc/source/stdlib/handmade/function-ast-variable_index-0xbf5517480aaeffb1.rst
@@ -0,0 +1,3 @@
+The slot the global ``variable`` holds in ``program``'s context: ``-1`` when the program
+never allocated it, ``-2`` when the allocation found it unused, its index in the
+context's global table otherwise.
diff --git a/doc/source/stdlib/handmade/function-debugapi-invoke_in_context-0x19a0ddf922c9ad95.rst b/doc/source/stdlib/handmade/function-debugapi-invoke_in_context-0x19a0ddf922c9ad95.rst
index 2bd510103e..c1406d48dc 100644
--- a/doc/source/stdlib/handmade/function-debugapi-invoke_in_context-0x19a0ddf922c9ad95.rst
+++ b/doc/source/stdlib/handmade/function-debugapi-invoke_in_context-0x19a0ddf922c9ad95.rst
@@ -1 +1 @@
-Calls a function in another context. Accepts a `Context` reference and either a function name (string), a `function` pointer, or a `lambda`, plus up to 10 extra arguments. Target functions must be marked `[export, pinvoke]`.
+Calls a function in another context. Accepts a `Context` reference and either a function name (string), a `function` pointer, or a `lambda`, plus up to 10 extra arguments. Target functions must be marked `[export, pinvoke]`. A name resolves against every function the context simulated, and a name held by more than one of them - an overload, a same-named function in another module - is an error rather than a pick.
diff --git a/doc/source/stdlib/handmade/function-debugapi-invoke_in_context-0x6a037dd8b7986835.rst b/doc/source/stdlib/handmade/function-debugapi-invoke_in_context-0x6a037dd8b7986835.rst
index 2bd510103e..c1406d48dc 100644
--- a/doc/source/stdlib/handmade/function-debugapi-invoke_in_context-0x6a037dd8b7986835.rst
+++ b/doc/source/stdlib/handmade/function-debugapi-invoke_in_context-0x6a037dd8b7986835.rst
@@ -1 +1 @@
-Calls a function in another context. Accepts a `Context` reference and either a function name (string), a `function` pointer, or a `lambda`, plus up to 10 extra arguments. Target functions must be marked `[export, pinvoke]`.
+Calls a function in another context. Accepts a `Context` reference and either a function name (string), a `function` pointer, or a `lambda`, plus up to 10 extra arguments. Target functions must be marked `[export, pinvoke]`. A name resolves against every function the context simulated, and a name held by more than one of them - an overload, a same-named function in another module - is an error rather than a pick.
diff --git a/doc/source/stdlib/handmade/function-debugapi-invoke_in_context-0xdc8aad5ea68674bf.rst b/doc/source/stdlib/handmade/function-debugapi-invoke_in_context-0xdc8aad5ea68674bf.rst
index 2bd510103e..c1406d48dc 100644
--- a/doc/source/stdlib/handmade/function-debugapi-invoke_in_context-0xdc8aad5ea68674bf.rst
+++ b/doc/source/stdlib/handmade/function-debugapi-invoke_in_context-0xdc8aad5ea68674bf.rst
@@ -1 +1 @@
-Calls a function in another context. Accepts a `Context` reference and either a function name (string), a `function` pointer, or a `lambda`, plus up to 10 extra arguments. Target functions must be marked `[export, pinvoke]`.
+Calls a function in another context. Accepts a `Context` reference and either a function name (string), a `function` pointer, or a `lambda`, plus up to 10 extra arguments. Target functions must be marked `[export, pinvoke]`. A name resolves against every function the context simulated, and a name held by more than one of them - an overload, a same-named function in another module - is an error rather than a pick.
diff --git a/doc/source/stdlib/handmade/function-debugapi-try_invoke_in_context-0x3b138017b43557c8.rst b/doc/source/stdlib/handmade/function-debugapi-try_invoke_in_context-0x3b138017b43557c8.rst
index 899bebddc8..b347ca0632 100644
--- a/doc/source/stdlib/handmade/function-debugapi-try_invoke_in_context-0x3b138017b43557c8.rst
+++ b/doc/source/stdlib/handmade/function-debugapi-try_invoke_in_context-0x3b138017b43557c8.rst
@@ -1 +1 @@
-The non-blocking `invoke_in_context`: tries the target context's lock instead of waiting on it. Returns `true` once the call ran, `false` - running nothing - when another thread holds that context, a debug-agent hook for instance. Same arguments as `invoke_in_context` by function name, up to 10 extra arguments; the target function must be marked `[export, pinvoke]`.
+The non-blocking `invoke_in_context`: tries the target context's lock instead of waiting on it. Returns `true` once the call ran, `false` - running nothing - when another thread holds that context, a debug-agent hook for instance. Same arguments as `invoke_in_context` by function name, up to 10 extra arguments; the target function must be marked `[export, pinvoke]`, and a name more than one of the context's functions holds is an error rather than a pick.
diff --git a/doc/source/stdlib/handmade/function-fio-register_module_group-0xbfd5bb5408ae846e.rst b/doc/source/stdlib/handmade/function-fio-register_module_group-0xbfd5bb5408ae846e.rst
new file mode 100644
index 0000000000..705f5f2076
--- /dev/null
+++ b/doc/source/stdlib/handmade/function-fio-register_module_group-0xbfd5bb5408ae846e.rst
@@ -0,0 +1 @@
+Registers ``member`` - a require path, spelled the way a ``require`` would spell it - under the group name ``group``, so ``require [group]`` in any module brings it in; ``guard``, when given, is the member's own guard, what ``require ?guard member`` would carry (a module name, or a path when it holds a ``/``), evaluated at every compile. A ``.das_module`` descriptor calls it beside ``register_native_path``; the scan records the row in the manifest and replays it on a warm start.
diff --git a/doc/source/stdlib/handmade/function-rtti-deserialize_program-0x15d628ec5a8ce8da.rst b/doc/source/stdlib/handmade/function-rtti-deserialize_program-0x15d628ec5a8ce8da.rst
new file mode 100644
index 0000000000..c03094257a
--- /dev/null
+++ b/doc/source/stdlib/handmade/function-rtti-deserialize_program-0x15d628ec5a8ce8da.rst
@@ -0,0 +1,5 @@
+Deserializes the next program from the stream and calls ``block`` with it, the way the
+two-argument form does, and gives the restored program ``access`` as its own: a stream carries
+no file access, and a ``require_module_now`` issued from the program's macros or ``[init]``
+walks through its program's access, so a reader that restores programs whose code requires
+modules late passes the access it would have compiled them with.
diff --git a/doc/source/stdlib/handmade/function-rtti-module_group_for_each_member-0x64b0c8812c8d58d6.rst b/doc/source/stdlib/handmade/function-rtti-module_group_for_each_member-0x64b0c8812c8d58d6.rst
new file mode 100644
index 0000000000..84bc5daaef
--- /dev/null
+++ b/doc/source/stdlib/handmade/function-rtti-module_group_for_each_member-0x64b0c8812c8d58d6.rst
@@ -0,0 +1,4 @@
+Calls ``block`` once per module registered under ``group`` - the require paths a
+``require [group]`` expands to, sorted by member path - and not at all for a group
+nothing joined. ``daslib/module_group``'s ``call_module_group`` reads the group through
+this call at macro time; a descriptor joins a group with ``register_module_group``.
diff --git a/doc/source/stdlib/handmade/module-cross_context.rst b/doc/source/stdlib/handmade/module-cross_context.rst
new file mode 100644
index 0000000000..c8eb0faaab
--- /dev/null
+++ b/doc/source/stdlib/handmade/module-cross_context.rst
@@ -0,0 +1,16 @@
+The CROSS_CONTEXT module requires a ``shared`` module after the prerequisite walk -
+from a macro, a simulate macro or an ``[init]``, never from a running script, which has
+no compile for the module to join - and calls into its macro context by function name,
+so a module that is expensive to bring up and only sometimes needed comes in at the point
+that decides it needs it.
+
+The function reached by name is ``[export]``\ ed - a macro context keeps only exported,
+``[init]``, ``[finalize]`` and macro-init functions - and it runs under the target
+context's lock, so it must not call back into the calling context; everything the caller
+needs comes back through the result pointer.
+
+All functions and symbols are in "cross_context" module, use require to get access to it.
+
+.. code-block:: das
+
+ require daslib/cross_context
diff --git a/doc/source/stdlib/handmade/module-module_group.rst b/doc/source/stdlib/handmade/module-module_group.rst
new file mode 100644
index 0000000000..dcf4e05651
--- /dev/null
+++ b/doc/source/stdlib/handmade/module-module_group.rst
@@ -0,0 +1,12 @@
+The MODULE_GROUP module calls into every member of a module group.
+``call_module_group("group", "entry", args...)`` expands at compile time to one
+``member::entry(args...)`` call per module registered under the group, sorted by
+member path, so the module that wrote ``require [group]`` reaches its members
+without naming them. A member without the entry is a compile error naming the
+call; a group nothing joined expands to nothing.
+
+All functions and symbols are in "module_group" module, use require to get access to it.
+
+.. code-block:: das
+
+ require daslib/module_group
diff --git a/doc/source/stdlib/handmade/typedef-ast-FunctionFlags.rst b/doc/source/stdlib/handmade/typedef-ast-FunctionFlags.rst
index fc223ab1ab..91e29ae4e2 100644
--- a/doc/source/stdlib/handmade/typedef-ast-FunctionFlags.rst
+++ b/doc/source/stdlib/handmade/typedef-ast-FunctionFlags.rst
@@ -9,7 +9,6 @@ Function moves return value.
Its an exported function.
Its an init function.
Function has address requested.
-Function is used.
Function is fast call.
Function has known side effects (user defined).
Function has to run at compile time.
diff --git a/doc/source/stdlib/handmade/typedef-ast-VariableFlags.rst b/doc/source/stdlib/handmade/typedef-ast-VariableFlags.rst
index 2d8f8aa021..d1a4f145fb 100644
--- a/doc/source/stdlib/handmade/typedef-ast-VariableFlags.rst
+++ b/doc/source/stdlib/handmade/typedef-ast-VariableFlags.rst
@@ -1,7 +1,6 @@
properties of the `Variable` object.
Variable is initialized via move <-
Variable is initialized via clone :=
-Variable is used
Variable is an alias for CMRES return value
Variable is marked as used (to suppress unused warnings)
Variable is a global shared variable
diff --git a/doc/source/stdlib/introduction.rst b/doc/source/stdlib/introduction.rst
index 662dede732..382f4e3546 100644
--- a/doc/source/stdlib/introduction.rst
+++ b/doc/source/stdlib/introduction.rst
@@ -97,6 +97,8 @@ Macros and metaprogramming
* :doc:`templates ` — ``decltype`` macro and ``[template]`` function annotation
* :doc:`templates_boost ` — template application helpers: variable/type replacement, hygienic names
* :doc:`macro_boost ` — miscellaneous macro manipulation utilities
+* :doc:`cross_context ` — a ``shared`` module required after the walk, and calls into its macro context
+* :doc:`module_group ` — ``call_module_group``: one call per member of a ``require [group]``
* :doc:`contracts ` — function argument contract annotations (``[expect_any_array]``, ``[expect_any_table]``, etc.)
* :doc:`apply ` — ``apply`` reflection pattern for struct field iteration
* :doc:`enum_trait ` — compile-time enumeration trait queries
diff --git a/doc/source/stdlib/sec_ast.rst b/doc/source/stdlib/sec_ast.rst
index 3580f14bfd..919f8525f1 100644
--- a/doc/source/stdlib/sec_ast.rst
+++ b/doc/source/stdlib/sec_ast.rst
@@ -16,6 +16,8 @@ templates, quoting, and macro infrastructure.
generated/templates_boost.rst
generated/quote.rst
generated/macro_boost.rst
+ generated/cross_context.rst
+ generated/module_group.rst
generated/typemacro_boost.rst
generated/ast_block_to_loop.rst
generated/ast_cursor.rst
diff --git a/include/daScript/ast/REVIEW.md b/include/daScript/ast/REVIEW.md
index 471a307ff7..0438e3396c 100644
--- a/include/daScript/ast/REVIEW.md
+++ b/include/daScript/ast/REVIEW.md
@@ -3,9 +3,22 @@
**Read `REVIEW_COMMON.md` (repo root) first - its contract binds this checklist.** Architecture doc:
`src/ast/ARCHITECTURE.md` (repo root).
-- **A diff that changes which module a bind registers into, or the name it registers under -
- `vectorHomeModule`, `typeFactory>::make`, `registerVectorFunctions`, the name a
- `ManagedVectorAnnotation` or `ManagedStructureAnnotation` takes (`ast_handle.h`) - bumps
- `LLVM_JIT_CODEGEN_VERSION` in `modules/dasLLVM/daslib/llvm_jit_run.das` (repo root), in the
- same change.** The JIT's DLL cache key folds the codegen version and each function's AST hash,
- never the module an extern lives in, so a cached DLL binds the old name and crashes on the hit.
+- **A diff that changes what a cached JIT DLL binds and the DLL key does not fold - the module a
+ bind registers into or the name it registers under (`vectorHomeModule`,
+ `typeFactory>::make`, `registerVectorFunctions`, the name a
+ `ManagedVectorAnnotation` or `ManagedStructureAnnotation` takes, `ast_handle.h`), or the field
+ layout of a C++ type das code reads through a binding - bumps `LLVM_JIT_CODEGEN_VERSION` in
+ `modules/dasLLVM/daslib/llvm_jit_plan.das` (repo root), in the same change.** The key folds
+ the codegen version and each function's AST hash, never the module an extern lives in nor a
+ bound type's offsets, so a cached DLL binds the old name or the old offset and crashes on the
+ hit.
+
+- **A diff that adds a field to `Function` or `Variable` (`ast.h`) holding something one
+ program's compile decides - whether the program uses it, the slot it holds in that program's
+ context, whether its JIT selected it - is a defect; the value goes into `Program`'s symbol
+ tables (`usedFunctions`, `functionIndices` and their variable twins, `jitSelected`)
+ instead.** A shared module's objects are one
+ instance for every program in the process, and a compile that runs inside another - a macro's
+ `compile`, a late `require`, the folding program - would otherwise overwrite the outer
+ program's answer mid-simulate. The per-compile fields that remain, and why each is tolerated,
+ are `src/ast/ARCHITECTURE.md` sec.4's.
diff --git a/include/daScript/ast/ast.h b/include/daScript/ast/ast.h
index b2bf8374af..394b3c83b8 100644
--- a/include/daScript/ast/ast.h
+++ b/include/daScript/ast/ast.h
@@ -304,7 +304,6 @@ namespace das
ExpressionPtr source = nullptr; // if its interator variable, this is where the source is
Expression * loop_source = nullptr; // weak ref to ExprFor::sources[i], used for read/write propagation
LineInfo at;
- int index = -1;
uint32_t stackTop = 0;
uint32_t extraLocalOffset = 0; // this is here for fake variables only
Module * module = nullptr;
@@ -315,7 +314,6 @@ namespace das
struct {
bool init_via_move : 1;
bool init_via_clone : 1;
- bool used : 1;
bool aliasCMRES : 1;
bool marked_used : 1;
bool global_shared : 1;
@@ -857,7 +855,6 @@ namespace das
vector arguments;
TypeDeclPtr result = nullptr;
ExpressionPtr body = nullptr;
- int32_t index = -1;
uint32_t totalStackSize = 0;
int32_t totalGenLabel = 0;
LineInfo at, atDecl;
@@ -890,7 +887,6 @@ namespace das
bool init : 1;
bool addr : 1;
- bool used : 1;
bool fastCall : 1;
bool knownSideEffects : 1;
bool hasToRunAtCompileTime : 1;
@@ -1111,6 +1107,12 @@ namespace das
DAS_API void setDeferredModuleLoader ( DeferredModuleLoader loader );
DAS_API DeferredModuleLoader getDeferredModuleLoader ();
DAS_API bool guardModuleAvailable ( const string & name );
+ struct ModuleGroupMember {
+ string member; // the require path
+ string guard; // `require ?guard member` when set - a module name, or a path when it holds a '/'
+ };
+ DAS_API void registerModuleGroupMember ( const string & group, const string & member, const string & guard = string() );
+ DAS_API vector getModuleGroupMembers ( const string & group );
class DAS_API Module {
public:
@@ -1666,8 +1668,26 @@ namespace das
unique_ptr thisModule;
ModuleLibrary library;
ModuleGroup * thisModuleGroup = nullptr;
+ FileAccessPtr access; // the access this program parses through
int totalFunctions = 0;
int totalVariables = 0;
+ // src/ast/ARCHITECTURE.md sec.4
+ das_hash_set usedFunctions;
+ das_hash_set usedVariables;
+ das_hash_map functionIndices;
+ das_hash_map variableIndices;
+ __forceinline bool isUsed ( const Function * fn ) const { return usedFunctions.find(fn) != usedFunctions.end(); }
+ __forceinline bool isUsed ( const Variable * var ) const { return usedVariables.find(var) != usedVariables.end(); }
+ __forceinline void setUsed ( const Function * fn, bool used ) { if ( used ) usedFunctions.insert(fn); else usedFunctions.erase(fn); }
+ __forceinline void setUsed ( const Variable * var, bool used ) { if ( used ) usedVariables.insert(var); else usedVariables.erase(var); }
+ __forceinline int32_t indexOf ( const Function * fn ) const { auto it = functionIndices.find(fn); return it != functionIndices.end() ? it->second : -1; }
+ __forceinline int32_t indexOf ( const Variable * var ) const { auto it = variableIndices.find(var); return it != variableIndices.end() ? it->second : -1; }
+ __forceinline void setIndex ( const Function * fn, int32_t index ) { functionIndices[fn] = index; }
+ __forceinline void setIndex ( const Variable * var, int32_t index ) { variableIndices[var] = index; }
+ das_hash_set jitSelected;
+ __forceinline bool isJitSelected ( const Function * fn ) const { return jitSelected.find(fn) != jitSelected.end(); }
+ __forceinline void setJitSelected ( const Function * fn, bool selected ) { if ( selected ) jitSelected.insert(fn); else jitSelected.erase(fn); }
+ __forceinline void clearJitSelection () { jitSelected.clear(); }
int newLambdaIndex = 1;
int inferPassesUsed = 0; // sum of inferTypesDirty inner-loop pass counts across all inferTypes calls (incl. restartInfer legs) for this module; reset by parseDaScript once per module-compile; used by per-module compile-time log
vector errors;
@@ -1733,6 +1753,9 @@ namespace das
TextWriter & logs, ModuleGroup & libGroup, CodeOfPolicies policies = CodeOfPolicies() );
DAS_CC_API ProgramPtr compileDaScriptSerialize ( const string & fileName, const FileAccessPtr & access,
TextWriter & logs, ModuleGroup & libGroup, CodeOfPolicies policies = CodeOfPolicies() );
+ // src/ast/ARCHITECTURE.md sec.3
+ DAS_CC_API Module * requireModuleNow ( const string & requireName, const FileAccessPtr & access,
+ TextWriter & logs, CodeOfPolicies policies = CodeOfPolicies() );
// optimization pass (compiler lib); runs after type inference
void optimizeProgram ( Program * program, TextWriter & logs, ModuleGroup & libGroup );
@@ -1821,7 +1844,6 @@ namespace das
uint64_t dataWalkerStringLimit = 0;
bool g_modulesInitialized = false;
-
static daScriptEnvironment *getBound();
static void setBound(daScriptEnvironment *bnd);
static daScriptEnvironment *exchangeBound(daScriptEnvironment *bnd);
diff --git a/include/daScript/ast/ast_serializer.h b/include/daScript/ast/ast_serializer.h
index 76d94e9b56..03a9cadb5a 100644
--- a/include/daScript/ast/ast_serializer.h
+++ b/include/daScript/ast/ast_serializer.h
@@ -132,6 +132,7 @@ namespace das {
bool builtinHashDrift = false; // last record failed on a builtin cumulative-hash mismatch (lazily populated builtin, e.g. dasbind) - deterministic per process, a rewrite changes nothing
bool quietCache = false;
uint64_t servedModules = 0;
+ int readingRecord = 0; // >0 while a record's payload deserializes (a late require nested in it cannot read)
string cutoffFile;
string cutoffReason;
bool policyMismatch = false;
@@ -170,8 +171,9 @@ namespace das {
vector> enumerationRefs;
// fieldRefs tuple contains: fieldptr, module, structname, fieldname
vector> fieldRefs;
- // parsedModules record: fileName, source content hash, source size, program, thisModule
- vector> parsedModules;
+ // parsedModules record: fileName, source content hash, source size, program, thisModule, the collector's require names
+ vector>> parsedModules;
+ size_t writtenModules = 0; // parsedModules already in the stream (writebackModules)
// tracking for shared modules
das_hash_set writingReadyModules;
bool ignoreEmptyExternal = false;
@@ -258,7 +260,7 @@ namespace das {
AstSerializer & serializeModule ( Module & module, bool already_exists );
static constexpr uint32_t getVersion () {
- return 205; // 205: a vector of a handled element streams under the element's module (204: the record header stamps the source by content hash; the policy stream carries every CodeOfPolicies field)
+ return 207; // 207: neither Function nor Variable flags carry a used bit, and neither streams an index (206: the record header carries the requires the parse took; 205: a vector of a handled element streams under the element's module; 204: the record header stamps the source by content hash; the policy stream carries every CodeOfPolicies field)
}
void serializeProgram ( ProgramPtr program, ModuleGroup & libGroup ) noexcept;
@@ -437,6 +439,11 @@ namespace das {
AstSerializerState * state,
const TBlock,const string> & block,
Context * context, LineInfoArg * at );
+ // the same, with the access the restored program's late requires walk through
+ void rtti_ast_serializer_deserialize_program_ex (
+ AstSerializerState * state, smart_ptr access,
+ const TBlock,const string> & block,
+ Context * context, LineInfoArg * at );
// Get serialized data from a writing serializer.
void rtti_ast_serializer_get_data (
diff --git a/include/daScript/ast/dyn_modules.h b/include/daScript/ast/dyn_modules.h
index 2eb753c387..c39243000c 100644
--- a/include/daScript/ast/dyn_modules.h
+++ b/include/daScript/ast/dyn_modules.h
@@ -33,12 +33,14 @@ DAS_CC_API bool require_dynamic_modules(smart_ptr file_access,
// src/ast/ARCHITECTURE.md sec.2
struct DynModuleManifestRow {
bool dynamic = false; // false: native path (a=mod_name, b=src, c=dst); true: dynamic module (a=path, b=cpp class, c=das name)
+ bool group = false; // group member (a=group, b=member, c=guard or empty)
string a, b, c;
int on_error = 0; // dynamic only: the RegisterOnError the descriptor used
};
DAS_API void begin_dynamic_module_recording();
DAS_API void end_dynamic_module_recording(vector & rows, bool & optOut);
DAS_API void replay_native_path(const char * mod_name, const char * src, const char * dst);
+DAS_API void replay_module_group(const char * group, const char * member, const char * guard);
DAS_API void replay_dynamic_module(const char * path, const char * cpp_class, int on_error);
DAS_API void defer_dynamic_module(const char * path, const char * cpp_class, int on_error, const char * das_name);
DAS_API bool load_deferred_dynamic_module(const char * das_name);
diff --git a/include/daScript/simulate/aot_builtin_ast.h b/include/daScript/simulate/aot_builtin_ast.h
index f399064b6b..0e762af32c 100644
--- a/include/daScript/simulate/aot_builtin_ast.h
+++ b/include/daScript/simulate/aot_builtin_ast.h
@@ -524,6 +524,21 @@ namespace das {
DAS_CC_API TypeDeclPtr parseMangledNameFn ( const char * txt, ModuleGroup & lib, Module * thisModule, Context * context, LineInfoArg * at );
DAS_CC_API void notInferred ( Function * func, Context * context, LineInfoArg * at );
DAS_API void collectDependencies ( FunctionPtr fun, const TBlock,TArray> & block, Context * context, LineInfoArg * line );
+ // src/ast/ARCHITECTURE.md sec.4
+ DAS_CC_API bool ast_is_function_used ( const Program * program, const Function * fn, Context * context, LineInfoArg * at );
+ DAS_CC_API bool ast_is_variable_used ( const Program * program, const Variable * var, Context * context, LineInfoArg * at );
+ DAS_CC_API int32_t ast_function_index ( const Program * program, const Function * fn, Context * context, LineInfoArg * at );
+ DAS_CC_API int32_t ast_variable_index ( const Program * program, const Variable * var, Context * context, LineInfoArg * at );
+ DAS_CC_API bool ast_is_function_used_sp ( smart_ptr_raw program, const Function * fn, Context * context, LineInfoArg * at );
+ DAS_CC_API bool ast_is_variable_used_sp ( smart_ptr_raw program, const Variable * var, Context * context, LineInfoArg * at );
+ DAS_CC_API int32_t ast_function_index_sp ( smart_ptr_raw program, const Function * fn, Context * context, LineInfoArg * at );
+ DAS_CC_API int32_t ast_variable_index_sp ( smart_ptr_raw program, const Variable * var, Context * context, LineInfoArg * at );
+ DAS_CC_API bool ast_is_jit_selected ( const Program * program, const Function * fn, Context * context, LineInfoArg * at );
+ DAS_CC_API void ast_set_jit_selected ( const Program * program, const Function * fn, bool selected, Context * context, LineInfoArg * at );
+ DAS_CC_API void ast_clear_jit_selection ( const Program * program, Context * context, LineInfoArg * at );
+ DAS_CC_API bool ast_is_jit_selected_sp ( smart_ptr_raw program, const Function * fn, Context * context, LineInfoArg * at );
+ DAS_CC_API void ast_set_jit_selected_sp ( smart_ptr_raw program, const Function * fn, bool selected, Context * context, LineInfoArg * at );
+ DAS_CC_API void ast_clear_jit_selection_sp ( smart_ptr_raw program, Context * context, LineInfoArg * at );
DAS_CC_API bool isExprLikeCall ( ExpressionPtr expr );
DAS_CC_API bool isExprConst ( ExpressionPtr expr );
DAS_CC_API bool isTempType ( TypeDeclPtr ptr, bool refMatters );
diff --git a/include/daScript/simulate/aot_builtin_rtti.h b/include/daScript/simulate/aot_builtin_rtti.h
index 17bc1882c7..2225b67960 100644
--- a/include/daScript/simulate/aot_builtin_rtti.h
+++ b/include/daScript/simulate/aot_builtin_rtti.h
@@ -138,6 +138,11 @@ namespace das {
const TBlock,const string> & block, Context * context, LineInfoArg * at );
DAS_API void rtti_builtin_compile_file(char * modName, smart_ptr access, ModuleGroup* module_group, const CodeOfPolicies & cop,
const TBlock, const string> & block, Context * context, LineInfoArg * lineinfo);
+ DAS_API void rtti_builtin_require_module_now ( char * name, smart_ptr access, const CodeOfPolicies & cop,
+ const TBlock & block, Context * context, LineInfoArg * at );
+ DAS_API bool rtti_has_module ( const char * name );
+ DAS_API bool rtti_has_macro_context ( Module * mod );
+ DAS_API Context & rtti_find_macro_context ( Module * mod, Context * context, LineInfoArg * at );
DAS_API void rtti_builtin_simulate ( const smart_ptr & program,
const TBlock,string> & block, Context * context, LineInfoArg * lineinfo );
diff --git a/modules/dasImgui/tests/test_grammar_canary.das b/modules/dasImgui/tests/test_grammar_canary.das
index b33b417cdd..3b44426f30 100644
--- a/modules/dasImgui/tests/test_grammar_canary.das
+++ b/modules/dasImgui/tests/test_grammar_canary.das
@@ -22,6 +22,8 @@ struct private CanarySection {
let CANARY_SECTIONS : array <- [
CanarySection(name = "options + optional-require guard",
snippet = "options gen2\noptions indenting = 4\n\nrequire ?zz_absent_guard daslib/algorithm\nrequire math\nrequire strings\nrequire daslib/stringify"),
+ CanarySection(name = "group require, bare and guarded",
+ snippet = "require [zz_absent_group]\nrequire ?zz_absent_guard [zz_absent_group] public"),
CanarySection(name = "enum with base type",
snippet = "enum CanMode : uint8 \{\n one\n two\n\}"),
CanarySection(name = "struct private + field default + @annotation",
diff --git a/modules/dasLLAMA/.das_module b/modules/dasLLAMA/.das_module
index acb28455ed..d7073074bb 100644
--- a/modules/dasLLAMA/.das_module
+++ b/modules/dasLLAMA/.das_module
@@ -12,4 +12,5 @@ def initialize(project_path : string) {
for (path in dasllama_paths) {
register_native_path("dasllama", "{path}", "{project_path}/dasllama/{path}.das")
}
+ register_module_group("llvm_code_generator", "dasllama/dasllama_gemm_register")
}
diff --git a/modules/dasLLAMA/ARCHITECTURE_ENGINE.md b/modules/dasLLAMA/ARCHITECTURE_ENGINE.md
index b818ca9eec..80173a6017 100644
--- a/modules/dasLLAMA/ARCHITECTURE_ENGINE.md
+++ b/modules/dasLLAMA/ARCHITECTURE_ENGINE.md
@@ -200,8 +200,8 @@ trunk-only and trunk+head images never collide and one image file serves both tr
registration. A hand-written tile that the generator could emit belongs in the generator.
- **`dasllama_tune.das`** - the per-box loop-hint tuner (`[tuned]` / `[dasllama_grid]`). Tuning
POLICY lives here; tuned VALUES live in the box's sidecar, never in source. The framework is
- OPTIONAL: the `llvm_tune` require is `?llvm`-guarded and every use of it sits behind
- `static_if (typeinfo module_exists(llvm_tune))`, so a build without dasLLVM opens no
+ OPTIONAL: `llvm_tune` comes through the `tune_framework` group, empty in a build that did not
+ configure dasLLVM, and every use of it sits behind `static_if (typeinfo module_exists(llvm_tune))`, so a build without dasLLVM opens no
sidecar, reads no policy env and reports nothing - `[tuned]` stamps the declared `fallback=`
(or `DEFAULT_PERM`) and the AST rewrite is unchanged. `box_profile_verdict_at` answers the
reason NAME rather than llvm_tune's enum for the same reason: a signature cannot be gated.
diff --git a/modules/dasLLAMA/benchmarks/lcpp_bench.das b/modules/dasLLAMA/benchmarks/lcpp_bench.das
index 17fb4b6c7d..f2c998359d 100644
--- a/modules/dasLLAMA/benchmarks/lcpp_bench.das
+++ b/modules/dasLLAMA/benchmarks/lcpp_bench.das
@@ -33,7 +33,7 @@ require daslib/jobque_profile // JOBQUE_PROFILING_ENABLED — --prof refuses o
require ?das_accelerate dasllama/dasllama_math_accelerate // the --accel leg; Apple-only C++ module
require ../performance/profile_common.das // profile_threads + affinity_on + bench records — the standing per-box methodology
require ?llvm dasllama/dasllama_exchange // nolint:STYLE030,LINT019 — OPTIONAL: --submit POSTs the stamped records to the exchange, which rides the tune framework
-require ?llvm llvm/daslib/llvm_tune // nolint:STYLE030,LINT019 — OPTIONAL: tune_app_script, the exe-vs-script discriminator behind --for-debug-purposes
+require [tune_framework] // nolint:STYLE030,LINT019 — OPTIONAL: tune_app_script, the exe-vs-script discriminator behind --for-debug-purposes
require daslib/jobque_boost
require daslib/clargs
require daslib/fio
diff --git a/modules/dasLLAMA/dasllama/dasllama_common.das b/modules/dasLLAMA/dasllama/dasllama_common.das
index 285c90aceb..7498e8f6cf 100644
--- a/modules/dasLLAMA/dasllama/dasllama_common.das
+++ b/modules/dasLLAMA/dasllama/dasllama_common.das
@@ -21,7 +21,7 @@ require dasllama/dasllama_math_default // registers the portable Q8·Q8 backend
require dasllama/dasllama_math_aarch64_neon // registers the arm64 SDOT Q8·Q8 backend at [init] under arm64 + JIT alone; compiles everywhere (daslib/aarch64_neon carries scalar fallbacks)
require dasllama/dasllama_math_gen // [init]-only: registers "arm64-gen" / "x64-gen" (the generated GEMM family) under the JIT alone; compiles everywhere - its reference bodies are the portable tier's kq kernels
require daslib/tune public // the tune annotations + the auto-policy pass, inert without dasLLVM; PUBLIC for the same reach as the line below
-require ?llvm llvm/daslib/llvm_tune public // re-export ONLY llvm_tune (not math_gen's whole public surface — its jobque_boost would flood requirers with ambiguities): tune_status reaches every app that requires this module
+require [tune_framework] public // re-export ONLY llvm_tune (not math_gen's whole public surface — its jobque_boost would flood requirers with ambiguities): tune_status reaches every app that requires this module
require ?das_metal dasllama/dasllama_metal_gemm // [init]-only: registers "metal" (the GPU prefill-GEMM batch donor). STRICT guard: the C++ das_metal module exists on Apple builds only, so non-Apple lanes never compile the Metal driver
require ?das_accelerate dasllama/dasllama_math_accelerate // [init]-only: arms the "+AMX" float-batch override when DASLLAMA_ACCEL=1. Same Apple-only guard shape as the metal driver
require dasllama/dasllama_gguf
diff --git a/modules/dasLLAMA/dasllama/dasllama_fat_start.das b/modules/dasLLAMA/dasllama/dasllama_fat_start.das
index 763955bc6f..f696e9fa75 100644
--- a/modules/dasLLAMA/dasllama/dasllama_fat_start.das
+++ b/modules/dasLLAMA/dasllama/dasllama_fat_start.das
@@ -10,7 +10,7 @@ require dasllama/dasllama_math // the runtime knob getters the snapshot record
require dasllama/dasllama_par // get_total_hw_jobs - the advisory "threads" entry
require daslib/json_boost
require daslib/strings_boost // nolint:STYLE030,LINT019 - join, used only inside the llvm_tune static_if half
-require ?llvm llvm/daslib/llvm_tune
+require [tune_framework]
require ?das_metal dasllama/dasllama_metal_prefill // metal_tensor_race (Apple static_if half)
require ?das_metal dasllama/dasllama_metal_kernels // metal_tensor_race_decode (Apple static_if half)
diff --git a/modules/dasLLAMA/dasllama/dasllama_gemm_register.das b/modules/dasLLAMA/dasllama/dasllama_gemm_register.das
index 520e401787..f9a2a4996a 100644
--- a/modules/dasLLAMA/dasllama/dasllama_gemm_register.das
+++ b/modules/dasLLAMA/dasllama/dasllama_gemm_register.das
@@ -7,13 +7,11 @@ module dasllama_gemm_register shared private
require dasllama/dasllama_lint public
require dasllama/dasllama_gemm_gen
-// The require?-target half of the generator wiring (gemm_generator_plan.md part 1):
-// llvm_user_modules does `require ?dasllama/dasllama_gemm_gen dasllama/dasllama_gemm_register`,
-// a PATH guard — this file (and through it the generator module) is pulled into the llvm-jit
-// context exactly when the dasllama package is mounted (the guard's own file resolves).
-// dasLLVM itself never hard-depends on dasLLAMA.
+//! The `llvm_code_generator` group member: the package's descriptor joins the group, and
+//! `llvm_user_modules` calls `register_llvm_code_generators` on every member, so dasLLVM
+//! never names dasLLAMA.
[macro_function]
-def public register_dasllama_gemm_code_generators {
+def public register_llvm_code_generators {
register_dasllama_gemm_generators()
}
diff --git a/modules/dasLLAMA/dasllama/dasllama_math_gen.das b/modules/dasLLAMA/dasllama/dasllama_math_gen.das
index 475a901694..8af8cf7df5 100644
--- a/modules/dasLLAMA/dasllama/dasllama_math_gen.das
+++ b/modules/dasLLAMA/dasllama/dasllama_math_gen.das
@@ -12,7 +12,7 @@ require dasllama/dasllama_math_default // dot_q8q8_f16s — the row-major tails
require dasllama/dasllama_repack // the grp repack family + kq panel unpack (pure transforms; the _gen/_bake twins below feed them numbers)
require daslib/f16_cvt // the kq reference bodies widen the f16 (s, o) scale pairs
require daslib/tune public // [tune]/[tune_perm]/[tune_companion] + [llvm_code], inert without dasLLVM; PUBLIC so the auto-policy infer pass reaches app roots that require this module
-require ?llvm llvm/daslib/llvm_tune public // PUBLIC so tune_status reaches app roots that require this module; witness guard: the one framework call below sits under static_if
+require [tune_framework] public // PUBLIC so tune_status reaches app roots that require this module; the one framework call below sits under static_if
require dasllama/dasllama_tune_scope // nolint:STYLE029,LINT019 — an ORDER dependency, not a symbol use: the dasllama [tune_scope] must be on the AST before this module's [tune] kernels pick their perms
require dasllama/dasllama_gemm_schema
require daslib/jobque_boost public
@@ -44,10 +44,9 @@ require math
// backends over the SAME
// ISA-agnostic slot traversals: "arm64-gen" (priority 25, the arm64 load-select tier since
// slice C) and "x64-gen" (priority 25, witness-gated — only selectable when the stamped
-// family emitted). dasllama_common pulls it in behind a path guard
-// (`require ?llvm`), so any Q8/mx4 model load on a gen-capable box runs the
-// generated family. The generators live in dasllama/dasllama_gemm_gen.das (llvm-jit context,
-// wired via llvm/daslib/llvm_user_modules.das).
+// family emitted). dasllama_common requires it, so any Q8/mx4 model load on a gen-capable box
+// runs the generated family. The generators live in dasllama/dasllama_gemm_gen.das (llvm-jit
+// context, reached through the llvm_code_generator group).
//! The layout companion (six-function stamp): the repack interleave (mr) the stamped tile reads.
//! Reference body = the reference tile's grp4; generator declines in lockstep with the tile
diff --git a/modules/dasLLAMA/dasllama/dasllama_tune.das b/modules/dasLLAMA/dasllama/dasllama_tune.das
index c3d006771a..4073967b2b 100644
--- a/modules/dasLLAMA/dasllama/dasllama_tune.das
+++ b/modules/dasLLAMA/dasllama/dasllama_tune.das
@@ -12,7 +12,7 @@ require daslib/ast_boost
require daslib/templates_boost
require daslib/json_boost // read_json / read_json_field (re-exports daslib/json)
require daslib/strings_boost // nolint:STYLE030,LINT019 - split/find, reached only by the no-framework fallback-chain scan
-require ?llvm llvm/daslib/llvm_tune // OPTIONAL: the per-app tune sidecar (path, staleness, kernels read)
+require [tune_framework] // OPTIONAL: the per-app tune sidecar (path, staleness, kernels read)
// ===== Tune spec =====
diff --git a/modules/dasLLAMA/dasllama/dasllama_tune_scope.das b/modules/dasLLAMA/dasllama/dasllama_tune_scope.das
index 2ffc1d69db..6051d06e9c 100644
--- a/modules/dasLLAMA/dasllama/dasllama_tune_scope.das
+++ b/modules/dasLLAMA/dasllama/dasllama_tune_scope.das
@@ -6,7 +6,7 @@ module dasllama_tune_scope shared public
require dasllama/dasllama_lint public
require daslib/tune public // the [tune_scope] annotation, inert without dasLLVM; PUBLIC so the auto-policy infer pass reaches app roots
-require ?llvm llvm/daslib/llvm_tune public // OPTIONAL; PUBLIC so the auto-policy infer pass and tune_status reach app roots
+require [tune_framework] public // OPTIONAL; PUBLIC so the auto-policy infer pass and tune_status reach app roots
require dasllama/dasllama_version // nolint:STYLE030 — version_of= below names DASLLAMA_RELEASE by string; the module must be in this compile
//! The scope reaches the AST before any covered module compiles - the scope apply refuses a covered
diff --git a/modules/dasLLAMA/performance/profile_common.das b/modules/dasLLAMA/performance/profile_common.das
index b37d93c582..70077ed9ac 100644
--- a/modules/dasLLAMA/performance/profile_common.das
+++ b/modules/dasLLAMA/performance/profile_common.das
@@ -21,7 +21,7 @@ require daslib/json public // JV(...) for logger field payloads
require daslib/json_boost // ?[] / ?? navigation — the sidecar provenance reader
require dasllama/dasllama_common public // QuantMode
require dasllama/dasllama_math // active_kernel_backend — the exec_fmt layout receipt
-require ?llvm llvm/daslib/llvm_tune // nolint:STYLE030,LINT019 — tune_status, the stamp provenance; empty without the framework
+require [tune_framework] // nolint:STYLE030,LINT019 — tune_status, the stamp provenance; empty without the framework
require model_specs public // the model-set table + its llm/official views
require box_ident // derive_box_name — the hardware-keyed box tag
diff --git a/modules/dasLLAMA/portable_tier_plan.md b/modules/dasLLAMA/portable_tier_plan.md
index d46ec14da5..14575a61a1 100644
--- a/modules/dasLLAMA/portable_tier_plan.md
+++ b/modules/dasLLAMA/portable_tier_plan.md
@@ -74,7 +74,8 @@ with LLVM on the host only; wasm32 cross is unsupported by design (pointer-width
borrowed from the IR generator (`pm1_of`, `vbmi_alphabet`, `vbmi_pack_word`) moved to
`dasllama_gemm_schema.das` for the same reason.
- **`require ?G target` names a guard module, never the target** (`ds2_parser.ypp:849-861`,
- `parser_impl.cpp:1225-1279`). The guard for everything LLVM is `?llvm`: dasLLVM is a pure-das
+ `parser_impl.cpp:1225-1279`). The guard for everything LLVM is `?llvm` - written by hand, or
+ carried by the `tune_framework` group row `require [tune_framework]` expands: dasLLVM is a pure-das
dasbind package, so it now carries one C++ witness module named `llvm`
(`modules/dasLLVM/src/dasLLVM.cpp`), compiled in exactly when the build is configured with
dasLLVM. The build configuration decides, never the files on disk - a wasm build never
@@ -492,15 +493,18 @@ the honest wasm candidates, and smaller is on the table.
with `-DDAS_LLVM_DISABLED=ON` still loaded the framework. The two scenarios that matter - a
wasm build, where no tuning may exist, and a console build (PS5 in dagor) with the LLVM
sources on disk for the PC build but configured out - both need the configure to decide. The
- `llvm` C++ witness module (see "Settled decisions") is that word; every guard is `?llvm`.
+ `llvm` C++ witness module (see "Settled decisions") is that word; every guard is `?llvm`, the
+ hand-written ones and the guard on the `tune_framework` group's member row alike.
- **RULED: a second trait.** `builtin_module_exists` on a shared das module flipped under
tool-driven compiles (`Module::requireEx` scans the process's promoted-module list, which a
nested compile never populates), so lint, ast-verify and the MCP checks audited the
no-framework arm of every guarded file. The ruling keeps `builtin_module_exists` as it was
(the process registry) and adds `typeinfo module_exists(X)`: the compiling program's own
- library, the same answer on both rails. Every path-guarded das target in the tree
- (`llvm_tune`, `llvm_code`, `dasllama_exchange`, `dasllama_gemm_gen`) now asks `module_exists`;
- C++-module guards keep `builtin_module_exists`. Gate: `tests/language/optional_require.das`.
+ library, the same answer on both rails. Every conditionally required das target in the tree
+ asks `module_exists` - `dasllama_exchange` under its hand-written `?llvm`, `llvm_tune` (and
+ `llvm_code` through it) under the `tune_framework` group's guarded row, `dasllama_gemm_gen`
+ through the `llvm_code_generator` group; C++-module guards keep `builtin_module_exists`.
+ Gate: `tests/language/optional_require.das`.
- Which small-LLM carrier gets the parity fixture for stages 1 and 2 (SmolLM2-135M already has
a cls_q8 parity cell, `test_parity.das:76`).
diff --git a/modules/dasLLVM/.das_module b/modules/dasLLVM/.das_module
index 9bb48822f8..1118a9442a 100644
--- a/modules/dasLLVM/.das_module
+++ b/modules/dasLLVM/.das_module
@@ -10,6 +10,8 @@ def initialize(project_path : string) {
"llvm_jit_intrin", "llvm_jit_common", "llvm_jit_lower", "llvm_dll_utils",
"llvm_jit_di",
"llvm_exe", "llvm_macro", "llvm_jit_cli", "llvm_jit_run", "llvm_aot",
+ "llvm_jit_plan", // the emitter-free half of the driver: plan, DLL key, probe, install
+ "llvm_jit_link", // the JIT entry: a DLL cache hit binds here, a miss brings llvm_macro in
"llvm_env", // [EnvConfig] environment-knob registry (ENVIRONMENT.md generates from it)
"llvm_cpu_class", // the CPU classes profiles are keyed by, and the DAS_JIT_BASELINE target
"llvm_code", // [llvm_code] user-side annotation
@@ -35,4 +37,6 @@ def initialize(project_path : string) {
// that silence is what `require ?llvm` reads as "this build has no LLVM"
register_dynamic_module("{project_path}/dasModuleLLVM.shared_module", "Module_LLVM")
}
+ // guarded on the witness, so the member expands exactly where `require ?llvm` would take it
+ register_module_group("tune_framework", "llvm/daslib/llvm_tune", "llvm")
}
diff --git a/modules/dasLLVM/ARCHITECTURE.md b/modules/dasLLVM/ARCHITECTURE.md
index 86367d1ca5..dff28fb1b8 100644
--- a/modules/dasLLVM/ARCHITECTURE.md
+++ b/modules/dasLLVM/ARCHITECTURE.md
@@ -2,21 +2,23 @@
The design document `REVIEW.md` cites. Numbered sections are the stable reference targets;
usage and installation live in `README.md`, the debugger rail and its roadmap in `DEBUGGING.md`.
-Companions: `ARCHITECTURE_TARGET_FEATURES.md` (CPU feature truth, the tier gates, the CPU classes),
-`ARCHITECTURE_DEBUG_INFO.md` (the `--jit-debug` DWARF rail - sec.12).
+Companions: `ARCHITECTURE_TARGET_FEATURES.md` (CPU feature truth, the tier gates, the CPU
+classes), `ARCHITECTURE_DEBUG_INFO.md` (the `--jit-debug` DWARF rail - sec.12) and
+`ARCHITECTURE_JIT_ENTRY.md` (the entry module, the emitter-free cache hit, the candidate-set key).
## 1. The jit backend pipeline
-`run_jit` (`llvm_jit_run.das`) drives one linear pipeline per program: **hash** (content-address
-the DLL - first per-function AOT hash plus the loop-hint/llvm-code folds), **init** (engine +
-target machine), **declare** (LLVM function declarations for the jit set), **probe** (open the
-cached DLL, compare per-function hashes), **irgen** (the das IR emitter over every function),
-**optimize** (the LLVM pass pipeline at the requested level, plus the opt-in IR dump and the
-post-opt verify), **emit+link** (artifact production - for the DLL path `write_dll` in
-`llvm_jit_common.das`, which itself splits into **emit-obj**, machine-code emission, and
-**link**, the lld-link spawn), **install** (resolve externs, instrument sim nodes), and
-**finalize** (engine teardown / state install). On a cache hit irgen, optimize, and emit+link
-are skipped and read as zero.
+`run_jit_linked` (`llvm_jit_link.das`) takes a DLL cache hit by itself - **hash**, **probe**,
+**install**, **finalize** - and hands every other run to `run_jit` (`llvm_jit_run.das`) in the
+emitter's macro context (`ARCHITECTURE_JIT_ENTRY.md`). `run_jit` drives one linear pipeline per
+program: **hash** (the DLL key over the candidate set: per-function AOT hashes plus the hint
+folds), **init** (engine + target machine), **declare** (LLVM function declarations for the jit
+set), **probe** (open the cached DLL, compare per-function hashes), **irgen** (the das IR emitter
+over every function), **optimize** (the LLVM pass pipeline at the requested level, plus the
+opt-in IR dump and the post-opt verify), **emit+link** (artifact production - for the DLL path
+`write_dll` in `llvm_jit_common.das`, itself **emit-obj**, machine-code emission, then **link**,
+the lld-link spawn), **install** (resolve externs, instrument sim nodes), and **finalize**
+(engine teardown / state install). On a hit irgen, optimize and emit+link read as zero.
### 1.1 The timing contract
@@ -64,13 +66,13 @@ on the tuner to mint them.
## 2. Codegen identity - the DLL cache
-Jit DLLs are content-addressed: `jit_dll_basename` folds the per-function AOT hashes,
-`LLVM_JIT_CODEGEN_VERSION`, the opt/size levels, prologue and debug-info flags, and the target
-triple - same inputs, same filename, cache hit. AST-level changes therefore self-invalidate
-through the function hashes; **emitter-level changes do not** - a change that alters generated
-machine code for identical inputs (IR generation, target-machine setup, `[llvm_code]`
-generators, the jit ABI) is invisible to the key and silently serves stale code from cache
-unless `LLVM_JIT_CODEGEN_VERSION` is bumped. Stamped `[llvm_code]` *arguments* are not
+Jit DLLs are content-addressed: `jit_dll_basename` (`llvm_jit_plan.das`) folds the candidate
+set's per-function AOT hashes (`ARCHITECTURE_JIT_ENTRY.md` sec.2), `LLVM_JIT_CODEGEN_VERSION`, the
+opt/size levels, prologue and debug-info flags, and the target triple - same inputs, same
+filename, cache hit. AST-level changes therefore self-invalidate through the function hashes;
+**emitter-level changes do not** - a change that alters generated machine code for identical
+inputs (IR generation, target-machine setup, `[llvm_code]` generators, the jit ABI) is invisible
+to the key and silently serves stale code from cache unless `LLVM_JIT_CODEGEN_VERSION` is bumped. Stamped `[llvm_code]` *arguments* are not
emitter-level: they fold into both cache keys per function (the hint folds), so a change that
merely re-selects which perm gets stamped - the `[tune]` machinery - self-invalidates with no
bump. "The jit call ABI" is the contract between the
diff --git a/modules/dasLLVM/ARCHITECTURE_JIT_ENTRY.md b/modules/dasLLVM/ARCHITECTURE_JIT_ENTRY.md
new file mode 100644
index 0000000000..0152d31ff4
--- /dev/null
+++ b/modules/dasLLVM/ARCHITECTURE_JIT_ENTRY.md
@@ -0,0 +1,89 @@
+# dasLLVM Architecture - the JIT entry and the cache hit
+
+Companion of `ARCHITECTURE.md` (sec.1 routes here). Contract: `ARCHITECTURE_COMMON.md` (repo
+root). This document carries how a program reaches the JIT and what a DLL cache hit costs.
+
+## 1. Three modules where there was one
+
+`daslib/just_in_time.das` - the module the host injects under `-jit` - requires
+`llvm_jit_link.das` and nothing of the emitter. The driver is split by what a cache hit needs:
+
+- `llvm_jit_plan.das` holds everything `run_jit` decides before it touches LLVM: the resolved
+ options (`JitPlan`, `make_jit_plan`), the candidate set, the DLL key (`jit_dll_basename`,
+ `jit_env_salt`, `LLVM_JIT_CODEGEN_VERSION`), the DLL probe (`probe_dll`, `get_dll_missing`),
+ the extern-slot resolution (`DllExternResolver`, `resolve_dll_externs`) and the sim-node
+ install (`install_dll_functions`). Its one use of the LLVM bindings is the three host
+ queries the key folds - default triple, host CPU name, host CPU features.
+- `llvm_jit_link.das` is the entry: the `jit_llvm` simulate macro, `run_jit_linked`, and the
+ `[finalize(late), no_jit]` finalizer that frees the jit state when the last sharing context
+ goes.
+- `llvm_macro.das` is the emitter's macro context, reached only by name: `run_jit_codegen`
+ wraps `run_jit` (`llvm_jit_run.das`), `free_jit_engine` disposes an in-memory engine. Nothing
+ requires it from a program.
+
+`run_jit_linked` computes the plan, and when the plan is a content-addressed or pinned DLL with
+candidates it opens the DLL and probes it. A hit binds in place: `mark_jit_selection` makes the
+program's jit selection (`Program::jitSelected`) the functions the DLL holds, `resolve_dll_externs` fills
+the DLL's extern slots, `install_dll_functions` points the sim nodes at the DLL's code, and
+`set_jit_state` records the handle. Everything else - a miss, an exe, a wasm, an AOT object,
+compile-only, an in-memory run, a static host - goes to the emitter:
+`macro_context_of("llvm/daslib/llvm_macro")` (`daslib/cross_context`) brings the module in
+through `require_module_now` on the first call and answers the context it already has after,
+and `invoke_in_context` runs `run_jit_codegen` there with the program and context pointers. The
+emitter recomputes the same plan - its inputs are the program and the command line - and takes
+the pipeline `ARCHITECTURE.md` sec.1 describes; on a warm run the emitter's ~40 modules are never
+read from the module cache and its macro contexts never simulated, which is the whole of the
+saving. The emitter's load is JIT work and prints its own `LLVM JIT time: emitter load` line; its modules
+are records of the script's own module cache (`src/ast/ARCHITECTURE.md` sec.3), written on the
+first miss and served on the next.
+
+## 2. The key folds the candidate set
+
+`jit_dll_basename` folds the AOT hashes of the CANDIDATES - every function the context holds
+that the policy selects, `[no_jit]` aside - not of the functions the emitter finally emits.
+The emitter's content gate (`DisableJitVisitor`) drops candidates it cannot lower, but the
+gate is a function of the candidates, the emitter version and the target, all of which the
+key folds, so one key still names one DLL. What the hit path gains is that it never runs the
+gate: `probe_dll` binds a candidate the DLL holds under a matching hash, leaves one the DLL
+lacks to the interpreter (the gate dropped it at codegen), and calls a candidate held under
+another hash, or a `[no_jit]` function the DLL holds, a miss. A DLL that binds nothing is a
+miss too: a pinned `-output` path is one name for every program pointed at it, and a DLL
+another program wrote holds none of this one's functions. A program with no candidates never
+loads the emitter - the link module logs the interpreted run and returns. An extern slot the
+process cannot fill on a hit - a builtin with no interop address here, a handled type's helper
+from a module this process lacks, process facts the key does not fold - is logged and handed to
+the emitter, whose content gate decides again and rewrites the DLL, where a panic would repeat
+on every run against the same name. `LLVM_JIT_CODEGEN_VERSION` 0x7d is the key's change of
+shape.
+
+The candidate set reads the CONTEXT, not the `used` flag alone: `get_function_by_mangled_name_hash`
+answers whether the simulated program holds the function. A module compiled into the process
+after this program simulated - the emitter a late require brings in - runs its own symbol-use
+passes over the shared modules it requires, and what they compute lives on its own `Program`
+(`src/ast/ARCHITECTURE.md` sec.4), so this program's answers stand; the context check is the
+second line, for the window inside one plan.
+
+## 3. The extern resolver needs no intrinsic table
+
+`DllExternResolver` walks each bound function and fills the DLL's extern slots: the interop
+address of every builtin called, the interpreter address of every callee the DLL does not hold
+(not in the program's jit selection), the handled-type and table helpers. The emitter's resolver used to consult
+`has_intrinsic` to skip calls it had lowered inline; the DLL already knows - a lowered call has no
+slot - so the resolver asks `set_glob_address`, which returns false for a slot the DLL lacks, and
+only a slot the DLL holds and nothing can fill is an error. The emitter and the hit path share the
+one resolver, and the visitor's borrowed fields (`dll`, `jit_context`, `uid`) carry
+`@do_not_delete` because the resolver is deleted after the walk and its objects live on.
+
+## 4. What stays out of the program's jitted set
+
+The finalizer is program code, so everything it reaches joins the program's used set and is a JIT
+candidate in every jitted program. `free_jit_context` is one function - a jitted program carries
+it and nothing else of the link module - `[no_jit]`, calling externs only (`get_module`,
+`find_macro_context`, `invoke_in_context`); a das
+helper on that path - `macro_context_of`, say - would put `daslib/cross_context`'s functions,
+blocks passed to C++ with a `LineInfoArg`, into every program's DLL, and the emitter cannot lower
+those. `llvm_exe.das`'s standalone-exe sweep skips `llvm_jit_link` the way it skips `llvm_macro`:
+an exe has no jit state to free, and the strict sweep refuses a `[no_jit]` function otherwise.
+A batch of programs in one process (dastest) exercises both: a program that fails inside the
+emitter must drain the codegen accumulators (`reset_codegen_accumulators`) on its way out, or the
+next program's fileinfo ctor asserts on names that belonged to the last one.
diff --git a/modules/dasLLVM/REVIEW.md b/modules/dasLLVM/REVIEW.md
index 39b015b436..0e3894be4d 100644
--- a/modules/dasLLVM/REVIEW.md
+++ b/modules/dasLLVM/REVIEW.md
@@ -31,23 +31,38 @@
the host, so a target-triple branch is checked only by the artifact built for that target.
- **A diff that adds work to, or moves work within, what `run_jit`
- (`daslib/llvm_jit_run.das`) executes - its own body or any callee - also prints an
- `LLVM JIT time:` number for that work: its own line, or the number of a phase that includes
- it, while that phase's line still prints** (phase inventory: `ARCHITECTURE.md` sec.1).
- Only work on the path that reaches the report is timed: option resolution before the first
- timer, log lines, and failure-path teardown are not.
+ (`daslib/llvm_jit_run.das`) or `run_jit_linked` (`daslib/llvm_jit_link.das`) executes - its
+ own body or any callee - also prints an `LLVM JIT time:` number for that work on every path
+ that executes it: its own line, or the number of a phase that includes it, while that phase's
+ line still prints; a number a second, independent computation of the same work prints (the
+ emitter's plan after the link module's) does not cover the first** (phase inventory:
+ `ARCHITECTURE.md` sec.1). Only work on the path that reaches the report is timed: option
+ resolution before the first timer, log lines, and failure-path teardown are not.
- **A change that can alter the machine code the JIT's DLL or split-obj cache serves back for
identical inputs - IR generation, target-machine setup, a `[llvm_code]` generator body, or the
call ABI the generated code binds: function signatures, the name scheme, the prologue, the
externs the install phase binds - bumps `LLVM_JIT_CODEGEN_VERSION`**
- (`daslib/llvm_jit_run.das`); selecting among existing generators' `[llvm_code]` arguments, the
+ (`daslib/llvm_jit_plan.das`); selecting among existing generators' `[llvm_code]` arguments, the
`[tune]` stamping, is not such a change. The caches are addressed by the AST hashes and this
constant, so such a change without the bump serves the old machine code back
(`ARCHITECTURE.md` sec.1.2).
+- **A diff that adds a `require` line naming a `[llvm_code]` generator module outside this
+ module - a package's, dasLLAMA's - to `daslib/llvm_user_modules.das` is a defect; the package
+ joins the `llvm_code_generator` group from its own descriptor instead.** The wiring module
+ names no package, so a build that does not carry the package registers nothing and compiles
+ unchanged; the generators this module ships stay named.
+
+- **A das function the jit finalizer reaches - `free_jit_context` (`daslib/llvm_jit_link.das`)
+ and anything it calls - is `[no_jit]` and calls only externs or other `[no_jit]` das
+ functions; a das helper on that path (`macro_context_of` and the rest of
+ `daslib/cross_context`) is a defect - call the externs directly.** The finalizer is program
+ code, so a jittable function on its path joins every jitted program's DLL (the block-passing
+ helpers cannot be lowered: `ARCHITECTURE_JIT_ENTRY.md` sec.4).
+
- **A diff that adds an environment or config input to a JIT cache key folds it inside
- `jit_env_salt` (`daslib/llvm_jit_run.das`), never directly into either JIT key - the DLL
+ `jit_env_salt` (`daslib/llvm_jit_plan.das`), never directly into either JIT key - the DLL
key or the split-obj key (`ARCHITECTURE.md` sec.2)** - salt feeds both keys, and a config
folded into one but not the other links stale objects. Inputs that vary per function set
(AOT hashes) fold into the key directly, not through the salt.
@@ -59,7 +74,8 @@
- **A change to a `[tune]`-family annotation is reviewed with `skills/tune.md`.**
-- **A change to the tune framework - `daslib/llvm_tune.das` or its tests - is reviewed with
+- **A change to the tune framework - `daslib/llvm_tune.das`, its tests, or the descriptor and
+ C++ rows that join it to a program (`.das_module`, `src/dasLLVM.cpp`) - is reviewed with
`skills/internal/llvm_tune_internals.md`.**
- **A test under `tests/` here whose child compiles through the front-end module cache - any
diff --git a/modules/dasLLVM/daslib/llvm_aot.das b/modules/dasLLVM/daslib/llvm_aot.das
index cef9d682b3..f5a80e81e6 100644
--- a/modules/dasLLVM/daslib/llvm_aot.das
+++ b/modules/dasLLVM/daslib/llvm_aot.das
@@ -72,7 +72,7 @@ def private emit_aot_register_ctor_dtor(types : PrimitiveTypes?; funcs : array;
- var uids : UidNodes?) : LLVMOpaqueValue? {
+ var uids : UidNodes?; prog : Program?) : LLVMOpaqueValue? {
// void das_aot_object_globinit_constructor(Context* ctx): ctx is used to
// resolve Func-value globs to the host's SimFunction* (das_aot_get_func_by_mnh).
let fnType = LLVMFunctionType(types.t_void, fixed_array(types.LLVMVoidPtrType()))
@@ -83,6 +83,7 @@ def public emit_aot_object_globinit(ctx : LLVMContextRef; mod : LLVMOpaqueModule
// Aliases the caller's uids; do NOT delete it (deletion cascades into that shared uid, which the
// caller still uses). It's a gc_node, reclaimed by the surrounding ast_gc_guard / process teardown.
var extern_resolver = new CollectExternVisitor(null, ctx, b, mod, types, uids, false, LlvmJitMode.LLVM_AOT)
+ extern_resolver.prog = prog // the selection a constructor is checked against is the program's
extern_resolver.aot_ctx_arg = LLVMGetParam(ctor, 0u)
make_visitor(*extern_resolver) $(adapter_resolve) {
ast_gc_guard() {
@@ -100,7 +101,7 @@ def public emit_aot_object_globinit(ctx : LLVMContextRef; mod : LLVMOpaqueModule
//! Build both offline-AOT-object ctors (the extern-resolver globinit + the das_aot_register ctor) and
//! return the (ctor, dtor) pair to append to llvm.global_ctors/dtors. Called by run_jit in emit_aot_object mode.
def public build_llvm_aot_ctors(ctx : LLVMContextRef; mod : LLVMOpaqueModule?; var types : PrimitiveTypes?; var funcs : array; var uids : UidNodes?; prog : Program?) : tuple {
- let glob_ctor = emit_aot_object_globinit(ctx, mod, types, funcs, uids)
+ let glob_ctor = emit_aot_object_globinit(ctx, mod, types, funcs, uids, prog)
return emit_aot_register_ctor_dtor(types, funcs, uids, prog, glob_ctor)
}
diff --git a/modules/dasLLVM/daslib/llvm_dll_utils.das b/modules/dasLLVM/daslib/llvm_dll_utils.das
index f00d265e99..c37d74231c 100644
--- a/modules/dasLLVM/daslib/llvm_dll_utils.das
+++ b/modules/dasLLVM/daslib/llvm_dll_utils.das
@@ -230,7 +230,7 @@ def public create_uid_nodes(prog : Program?; funcs : array; gen_exe
if (gen_exe) {
prog |> for_each_module() $(m) {
m |> for_each_global() $(var glob) {
- if (glob.init != null && glob.flags.used) {
+ if (glob.init != null && is_used(prog, glob)) {
visit_expression(glob.init, adapter)
}
}
diff --git a/modules/dasLLVM/daslib/llvm_exe.das b/modules/dasLLVM/daslib/llvm_exe.das
index dc2481ce61..af2346994e 100644
--- a/modules/dasLLVM/daslib/llvm_exe.das
+++ b/modules/dasLLVM/daslib/llvm_exe.das
@@ -42,6 +42,7 @@ bitfield TabOperation {
class public CollectExternVisitor : AstVisitor {
uid : UidNodes?
+ @do_not_delete prog : Program?
//! every function registered with the exe's context, once each - the set its function lookup carries
registered : array>
@@ -254,14 +255,15 @@ class public CollectExternVisitor : AstVisitor {
let fnmna = uid.get_dll_fn_name_ptr(fn)
let fn_publ = LLVMGetNamedFunction(g_mod, fnmna.publ())
if (fn_publ == null) return null
- if (!(registered_index |> key_exists(fn.index))) {
- registered_index |> insert(fn.index)
- registered |> push((index = fn.index, mnh = fn.getMangledNameHash, name = string(fn.name)))
+ let findex = function_index(prog, fn)
+ if (!(registered_index |> key_exists(findex))) {
+ registered_index |> insert(findex)
+ registered |> push((index = findex, mnh = fn.getMangledNameHash, name = string(fn.name)))
}
let fn_ptr_cast = LLVMBuildPointerCast(builder, fn_publ, types.LLVMVoidPtrType(), "")
var reg_args = fixed_array(
standalone_ctx,
- types.ConstI64(fn.index |> uint64),
+ types.ConstI64(findex |> uint64),
get_string_constant_ptr(builder, string(fn.name)),
get_string_constant_ptr(builder, fn |> get_mangled_name()),
types.ConstI64(fn.getMangledNameHash),
@@ -499,7 +501,7 @@ class public CollectExternVisitor : AstVisitor {
}
def override preVisitExprMakeStructIndex(expr : ExprMakeStruct?; index : int; last : bool) : void {
- if (expr.constructor != null && !expr.constructor.moreFlags.requestJit) {
+ if (expr.constructor != null && !is_jit_selected(prog, expr.constructor)) {
register_function(expr.constructor)
}
}
@@ -620,6 +622,7 @@ def collect_external_functions(standalone_context : LLVMOpaqueValue?; ctx : LLVM
dynamic_modules : table;
prog : Program?; register_all_modules : bool = false) : tuple>> {
var extern_resolver = new CollectExternVisitor(standalone_context, ctx, builder, mod, types, uids, register_all_modules, LlvmJitMode.EXE)
+ extern_resolver.prog = prog
extern_resolver.dynamic_modules := dynamic_modules
make_visitor(*extern_resolver) $(adapter_resolve) {
ast_gc_guard() {
@@ -630,7 +633,7 @@ def collect_external_functions(standalone_context : LLVMOpaqueValue?; ctx : LLVM
prog |> for_each_module() $(m) {
if (m.moduleFlags.builtIn) return ;
m |> for_each_global() $(var glob) {
- if (glob.init != null && glob.flags.used) {
+ if (glob.init != null && is_used(prog, glob)) {
visit_expression(glob.init, adapter_resolve)
}
}
@@ -1031,9 +1034,9 @@ def public inject_main(program_context : Context?; ctx : LLVMContextRef; // noli
var any_pinvoke = false
var used_modules : table
prog |> for_each_module() $(m) {
- if (m.name == "llvm_func" || m.name == "llvm_macro") return
+ if (m.name == "llvm_func" || m.name == "llvm_macro" || m.name == "llvm_jit_link") return
m |> for_each_function("") $(fun) {
- if (fun.flags.used) {
+ if (is_used(prog, fun)) {
if (!fun.flags.builtIn && fun.moreFlags.requestNoJit) {
has_no_jit = true
to_log(LOG_ERROR, "LLVM EXE: function '{fun.name}' is marked no_jit but standalone exe requires all functions to be JIT-compiled\n")
diff --git a/modules/dasLLVM/daslib/llvm_jit.das b/modules/dasLLVM/daslib/llvm_jit.das
index c0e13463d7..1b9cae4e18 100644
--- a/modules/dasLLVM/daslib/llvm_jit.das
+++ b/modules/dasLLVM/daslib/llvm_jit.das
@@ -23,6 +23,7 @@ require llvm/daslib/llvm_jit_code
require llvm/daslib/llvm_cpu_class
require llvm/daslib/llvm_user_modules // nolint:STYLE030,LINT019 - side-effect require: [llvm_code] generator modules must be in THIS module's closure so their [init] registration runs in the context that reads the registry; STYLE030 is policy-gated, so the keep only fires under a require-analysis run
require llvm/daslib/llvm_dll_utils
+require llvm/daslib/llvm_jit_plan
require daslib/ast_boost
require daslib/templates_boost
require daslib/macro_boost
@@ -196,17 +197,6 @@ def get_function_addr(var ctx : Context?; func : Function?) {
return unsafe(reinterpret(MNH_ADDR))
}
-// Check if a function is a late-bound dasbind function (null builtin address, resolvable at runtime)
-[macro_function]
-def is_dasbind_late(func : Function?) : bool {
- if (func == null || !func.flags.builtIn || func._module.name != "dasbind") return false
- var hit = false
- peek(func.name) $(s) {
- hit = starts_with(s, "__dasbind__")
- }
- return hit
-}
-
// Codegen mode: how function/global symbols are emitted. Distinct from the run_jit output concern
@@ -246,55 +236,10 @@ struct TableWalkState {
stride : int // value stride in bytes (values lane; keys GEP by element type)
}
-def private loop_source_is_jitable(src : ExpressionPtr) : bool {
- assume srcT = src._type
- if (srcT == null) {
- return false
- }
- return (srcT.isRange || srcT.baseType == Type.tFixedArray || srcT.isGoodArrayType
- || srcT.isIterator || srcT.isString || handle_each_is_jitable(src))
-}
-
-def public handle_each_is_jitable(src : ExpressionPtr) : bool {
- assume srcT = src._type
- if (srcT == null || !srcT.isHandle || srcT.annotation == null) {
- return false
- }
- return get_jit_each(srcT.annotation) != null
-}
-
def private is_int64_or_uint64(t : Type) : bool {
return t == Type.tInt64 || t == Type.tUInt64
}
-//! The jit passes a handled index in its own C ABI, so the index has to be a primitive the
-//! emitter can put in a register. Anything else (a typed id, a struct key) gets no address and
-//! keeps the interpreter, whatever the annotation offers. Returns -1 when not passable.
-def public jit_index_base_type(ixT : TypeDeclPtr) : int {
- if (ixT == null || ixT.isRef) {
- return -1
- }
- let bt = ixT.baseType
- if (bt == Type.tInt || bt == Type.tUInt || bt == Type.tInt64 || bt == Type.tUInt64 ||
- bt == Type.tInt8 || bt == Type.tUInt8 || bt == Type.tInt16 || bt == Type.tUInt16 ||
- bt == Type.tFloat || bt == Type.tString) {
- return int(bt)
- }
- return -1
-}
-
-def public handle_at_is_jitable(expr : ExprAt?) : bool {
- assume subExprT = expr.subexpr._type
- if (!subExprT.isHandle || subExprT.annotation == null) {
- return false
- }
- let ibt = jit_index_base_type(expr.index._type)
- if (ibt < 0) {
- return false
- }
- return get_jit_at(subExprT.annotation, ibt) != null
-}
-
[macro]
class public LlvmJitVisitor : AstVisitor {
adapter : VisitorAdapter?
@@ -319,6 +264,7 @@ class public LlvmJitVisitor : AstVisitor {
loop_stack : array
block_stack : array
jit_context : Context?
+ @do_not_delete prog : Program?
function_entry : LLVMOpaqueBasicBlock?
function_body : LLVMOpaqueBasicBlock?
monad2block : table
@@ -349,7 +295,8 @@ class public LlvmJitVisitor : AstVisitor {
ldu_hint : bool
attributes : Attributes?
- def LlvmJitVisitor(ctx : Context?; types_ : PrimitiveTypes?; uids : UidNodes?; flags_ : LlvmJitFlags; mode_ : LlvmJitMode; dib : LLVMOpaqueDIBuilder?; attrs : Attributes?) {
+ def LlvmJitVisitor(prog_ : Program?; ctx : Context?; types_ : PrimitiveTypes?; uids : UidNodes?; flags_ : LlvmJitFlags; mode_ : LlvmJitMode; dib : LLVMOpaqueDIBuilder?; attrs : Attributes?) {
+ prog = prog_
jit_context = ctx
types = types_
g_builder = LLVMCreateBuilderInContext(g_ctx)
@@ -1901,7 +1848,7 @@ class public LlvmJitVisitor : AstVisitor {
}
}
} else {
- if (expr.func.moreFlags.requestJit) {
+ if (is_jit_selected(prog, expr.func)) {
var params <- build_call_params(expr)
params |> push(get_context_param())
if (cmresEval != null) {
@@ -4546,7 +4493,7 @@ class public LlvmJitVisitor : AstVisitor {
// ExprMakeStruct
def callConstructor(expr : ExprMakeStruct?; cmresEval : LLVMOpaqueValue?) {
assume expr_func = expr.constructor
- if (expr_func.moreFlags.requestJit) {
+ if (is_jit_selected(prog, expr_func)) {
var params <- [
get_context_param(),
LLVMBuildPointerCast(g_builder, cmresEval, types.LLVMVoidPtrType(), "")
@@ -6744,7 +6691,7 @@ class public LlvmJitVisitor : AstVisitor {
}
def make_block_function(expr : ExprMakeBlock?; captured : array; first_field_index : int; captureType : LLVMOpaqueType?) : tuple {
- var astVisitor = new LlvmJitVisitor(jit_context, types, uid, flags, mode, g_di_builder, attributes)
+ var astVisitor = new LlvmJitVisitor(prog, jit_context, types, uid, flags, mode, g_di_builder, attributes)
var result : LLVMOpaqueValue?
var impl : LLVMOpaqueValue?
make_visitor(*astVisitor) $(mv_adapter) {
@@ -7788,285 +7735,9 @@ def private add_ctor_dtor_function(ctor_dtor : array
- uid : UidNodes?
-
- def override canVisitWithAliasSubexpression(expr : ExprAssume?) : bool {
- return false
- }
-
- def override preVisitFunction(var fun : FunctionPtr) : void {
- uid.reset(fun)
- thisFunc = fun
- }
-
- def override visitFunction(fun : FunctionPtr) : FunctionPtr {
- thisFunc = null
- uid.reset(null)
- return fun
- }
-
-
- def ResolveExternVisitor(var _dll : DLLHandle?; var _types : PrimitiveTypes?; ctx : Context?; uids : UidNodes?) {
- dll = _dll
- types = _types
- jit_context = ctx
- uid = uids
- }
-
- def make_call(expr : ExprCallFunc?) {
- if (expr.func == null || has_intrinsic(expr)) return
-
- if (expr.func.flags.builtIn) {
- let FUNC_PTR = get_builtin_function_address(expr.func)
- var is_dasbind = false
- peek(expr.func.name) $(s) {
- is_dasbind = find(s, "__dasbind__") >= 0
- }
- if (FUNC_PTR == null && !is_dasbind) {
- failed_E(expr, "missing interop function pointer for {expr.func._module.name}::{expr.func.name} // {get_mangled_name(expr.func)}")
- }
- dll.set_glob_address(uid.get_dll_fn_name_ptr(expr.func), FUNC_PTR)
- } else {
- if (!expr.func.moreFlags.requestJit) {
- var MNH_ADDR = get_function_addr(jit_context, expr.func)
- dll.set_glob_address(uid.get_dll_fn_name_ptr(expr.func), MNH_ADDR)
- }
- }
- }
- def override preVisitExprNew(expr : ExprNew?) {
- if (expr.typeexpr.isHandle) {
- var new_handle_ptr = get_jit_new(expr.typeexpr.annotation)
- if (new_handle_ptr == null) {
- failed_T(expr.typeexpr, "missing new`handle function pointer for {describe(expr.typeexpr)}")
- return
- }
- dll.set_glob_address(uid.get_new(expr.typeexpr), new_handle_ptr)
- } else {
- if (expr.typeexpr.baseType == Type.tFixedArray) {
- // the handle gate lives on the chain element (the head is tFixedArray);
- // borrow-walk — this pass runs outside ast_gc_guard, clones would leak
- var elemT = expr.typeexpr
- while (elemT.baseType == Type.tFixedArray && elemT.firstType != null) {
- elemT = elemT.firstType
- }
- if (elemT.baseType == Type.tHandle) {
- var new_handle_ptr = get_jit_new(elemT.annotation)
- if (new_handle_ptr == null) {
- failed_T(expr.typeexpr, "missing new`handle function pointer for {describe(expr.typeexpr)}")
- return
- }
- dll.set_glob_address(uid.get_new(expr.typeexpr), new_handle_ptr)
- } else {
- if (expr.initializer) {
- make_call(expr)
- }
- }
- } else {
- if (expr.initializer) {
- make_call(expr)
- }
- }
- }
- }
-
- def override preVisitExprCall(expr : ExprCall?) {
- make_call(expr)
- }
-
- def override preVisitExprOp2(var expr : ExprOp2?) {
- if (isExprOp2_Func(expr)) {
- make_call(expr)
- }
- }
-
- def override preVisitExprOp1(var expr : ExprOp1?) {
- if (isExprOp1_Func(expr)) {
- make_call(expr)
- }
- }
-
- def override preVisitExprDelete(var expr : ExprDelete?) {
- assume subexpr = expr.subexpr
- assume subT = expr.subexpr._type
- if (subT.isPointer) {
- assume firstT = expr.subexpr._type.firstType
- if (firstT.isHandle) {
- var delete_handle_ptr = get_jit_delete(firstT.annotation)
- if (delete_handle_ptr == null) {
- failed_T(firstT, "missing delete`handle function pointer for {describe(firstT)}")
- return
- }
- dll.set_glob_address(uid.get_delete_type(firstT), delete_handle_ptr)
- }
- }
- }
-
- def override preVisitExprClone(expr : ExprClone?) : void {
- assume leftT = expr.left._type
- if (leftT.isHandle) {
- var clone_handle_ptr = get_jit_clone(leftT.annotation)
- if (clone_handle_ptr == null) {
- failed_T(leftT, "missing clone`handle function pointer for {describe(leftT)}")
- return
- }
- dll.set_glob_address(uid.get_clone_type(leftT), clone_handle_ptr)
- }
- }
-
- def override preVisitExprAddr(expr : ExprAddr?) {
- var MNH_ADDR = get_function_addr(jit_context, expr.func)
- dll.set_glob_address(uid.get_addr(expr), MNH_ADDR)
- }
-
- def override preVisitExprMakeStructIndex(expr : ExprMakeStruct?; index : int; last : bool) : void {
- // one time is enough?
- if (expr.constructor != null && !expr.constructor.moreFlags.requestJit) {
- var MNH_ADDR = get_function_addr(jit_context, expr.constructor)
- dll.set_glob_address(uid.get_ctor(expr), MNH_ADDR)
- }
- }
-
- def override preVisitExprAscend(expr : ExprAscend?) : void {
- if (expr.subexpr._type.isHandle) {
- assume firstT = expr.subexpr._type
- var new_handle_ptr = get_jit_new(firstT.annotation)
- if (new_handle_ptr == null) {
- failed_T(firstT, "missing new`handle function pointer for {describe(firstT)}")
- return
- }
- dll.set_glob_address(uid.get_ascend_new(expr), new_handle_ptr)
- }
- }
-
- def override preVisitExprMakeBlock(expr : ExprMakeBlock?) : void {
- thisBlock |> push <| unsafe(reinterpret(expr._block))
- }
-
- def override visitExprMakeBlock(expr : ExprMakeBlock?) : ExpressionPtr {
- thisBlock |> pop()
- return expr
- }
-
- def private get_table_t(subexprT : TypeDeclPtr) {
- assume firstT = subexprT.firstType
- if (firstT.isWorkhorseType) {
- return firstT.baseType
- } else {
- var uvt = get_underlying_value_type(firstT)
- return uvt.baseType
- }
- }
-
- // Pre-bind the JIT table-fn globals for the standalone-exe / cache-reload path. String
- // keys route through the precomputed-hash helpers (see build_table_find/at), so bind
- // those globals instead of the per-type ones.
- def bind_jit_table_find_glob(t : Type) {
- // String find is fully inline (no glob). Non-string binds the per-type C++ find glob — still
- // used by non-workhorse keys (vectors / structs) that route through build_table_find_call.
- if (t != Type.tString) {
- dll.set_glob_address(DllName(FN_JIT_TABLE_FIND(t)), get_jit_table_find(unsafe(reinterpret(t))))
- }
- }
- def bind_jit_table_at_glob(t : Type) {
- if (t == Type.tString) {
- dll.set_glob_address(DllName(FN_JIT_STRING_TABLE_AT_WITH_HASH()), get_jit_string_table_at_with_hash())
- dll.set_glob_address(DllName(FN_JIT_STRING_TABLE_AT_AFTER_PACKED_MISS()), get_jit_string_table_at_after_packed_miss())
- } else {
- // Non-string keys are open-addressed: the inline probe handles find/at hits and routes a
- // miss to the regular C++ at (FN_JIT_TABLE_AT) — no packed after-miss helper needed.
- dll.set_glob_address(DllName(FN_JIT_TABLE_AT(t)), get_jit_table_at(unsafe(reinterpret(t))))
- }
- }
-
- def override preVisitExprAt(expr : ExprAt?) {
- assume subExprT = expr.subexpr._type
- if (subExprT.isGoodTableType) {
- let t = get_table_t(subExprT)
- bind_jit_table_at_glob(t)
- } elif (handle_at_is_jitable(expr)) {
- assume ixT = expr.index._type
- dll.set_glob_address(uid.get_at_type(subExprT, ixT),
- get_jit_at(subExprT.annotation, jit_index_base_type(ixT)))
- }
- }
-
- def override preVisitExprForSource(var expr : ExprFor?, var source : ExpressionPtr, last : bool) : void {
- if (handle_each_is_jitable(source)) {
- dll.set_glob_address(uid.get_each_type(source._type), get_jit_each(source._type.annotation))
- }
- }
-
- def override preVisitExprSetInsert(expr : ExprSetInsert?) {
- let t = get_table_t(expr.arguments[0]._type)
- bind_jit_table_at_glob(t)
- }
-
- def override preVisitExprKeyExists(expr : ExprKeyExists?) {
- let t = get_table_t(expr.arguments[0]._type)
- bind_jit_table_find_glob(t)
- }
-
- def override preVisitExprFind(expr : ExprFind?) {
- let t = get_table_t(expr.arguments[0]._type)
- bind_jit_table_find_glob(t)
- }
-
- def override preVisitExprSafeAt(expr : ExprSafeAt?) {
- assume subExprT = expr.subexpr._type
- if (subExprT.isGoodTableType || (subExprT.isPointer && subExprT.firstType.isGoodTableType)) {
- var etype = subExprT.isPointer ? subExprT.firstType : subExprT
- let t = get_table_t(etype)
- bind_jit_table_find_glob(t)
- }
- }
-
- def override preVisitExprErase(expr : ExprErase?) {
- let t = get_table_t(expr.arguments[0]._type)
- dll.set_glob_address(DllName(FN_JIT_TABLE_ERASE(t)), get_jit_table_erase(unsafe(reinterpret(t))))
- }
-
-}
-
-def public get_dll_missing(fns : array; disabled : array; var dll : DLLHandle?; var uids : UidNodes?) {
- verify(!(is_in_completion() || is_compiling_macros() || is_folding()))
-
- var missing = array()
- missing |> reserve(length(fns))
- var disabled_exists = array()
- if (dll.handle == null) {
- for (fn in fns) {
- missing |> push(unsafe(reinterpret(fn)))
- }
- } else {
- for (fn in fns) {
- let dll_name = uids.get_dll_fn_name(fn)
- var written_hash = dll.get_function_hash(dll_name)
- // debug("dll_function_aot_hash: {dll_function_aot_hash}")
- if (written_hash == null || get_function_aot_hash(fn) != *written_hash) {
- missing |> push(unsafe(reinterpret(fn)))
- }
- }
- for (fn in disabled) {
- let dll_name = uids.get_dll_fn_name(fn)
- var written_hash = dll.get_function_hash(dll_name)
- // debug("dll_function_aot_hash: {dll_function_aot_hash}")
- if (written_hash != null) {
- disabled_exists |> push(unsafe(reinterpret(fn)))
- }
- }
- }
- return <- (missing, disabled_exists)
-}
-
// Main function. Generates LLVM IR for `FunctionPtr`.
[macro_function]
-def public generate_llvm(ctx : Context?; types : PrimitiveTypes?; uids : UidNodes?;
+def public generate_llvm(prog : Program?; ctx : Context?; types : PrimitiveTypes?; uids : UidNodes?;
attrs : Attributes?; fn : FunctionPtr; flags : LlvmJitFlags; mode : LlvmJitMode) : bool {
verify(!(is_in_completion() || is_compiling_macros() || is_folding()))
@@ -8079,7 +7750,7 @@ def public generate_llvm(ctx : Context?; types : PrimitiveTypes?; uids : UidNode
// debug("initializer hash for {dll_name.publ()}: {aot_hash}")
globalVar |> LLVMSetInitializer(types.ConstI64(aot_hash))
}
- var astVisitor = new LlvmJitVisitor(ctx, types, uids, flags, mode, null, attrs)
+ var astVisitor = new LlvmJitVisitor(prog, ctx, types, uids, flags, mode, null, attrs)
var handled_by_generator = false
if (find_llvm_code_annotation(fn) != null) {
ast_gc_guard() {
@@ -8119,7 +7790,7 @@ def public generate_globals_initialization_fn(ctx : Context?; prog : Program?;
// globals — they belong to the owner, and writing them corrupts the parent's shared buffer. Matches
// the interpreter's `if (sharedOwner || !pv.shared)` gate (context.cpp Context::runInitScript).
let attrs = new Attributes(g_ctx)
- var visitor = new LlvmJitVisitor(ctx, types, uids, flags, mode, null, attrs)
+ var visitor = new LlvmJitVisitor(prog, ctx, types, uids, flags, mode, null, attrs)
var globals_fn : LLVMOpaqueValue?
var globals_fn_type : LLVMOpaqueType?
make_visitor(*visitor) $(adapter) {
@@ -8149,7 +7820,7 @@ def public generate_globals_initialization_fn(ctx : Context?; prog : Program?;
prog |> for_each_module() $(mod) {
mod |> for_each_global() $(glob) {
- if (glob.init != null && glob.flags.used) {
+ if (glob.init != null && is_used(prog, glob)) {
// A clone is not the shared owner: leave shared globals as the
// parent's (re-init would corrupt the shared buffer). See skip_shared.
if (skip_shared && glob.flags.global_shared) { return }
diff --git a/modules/dasLLVM/daslib/llvm_jit_common.das b/modules/dasLLVM/daslib/llvm_jit_common.das
index 1c8c1cbb78..ad97f866aa 100644
--- a/modules/dasLLVM/daslib/llvm_jit_common.das
+++ b/modules/dasLLVM/daslib/llvm_jit_common.das
@@ -12,6 +12,7 @@ module llvm_jit_common shared private
require llvm/daslib/llvm_boost
require llvm/daslib/llvm_dll_utils
+require llvm/daslib/llvm_jit_plan public
require llvm/daslib/llvm_targets
require llvm/daslib/llvm_dsl
@@ -34,9 +35,6 @@ let public LLVM_DEBUG_LINE_TRACES = false
let public LLVM_JIT_LOG = false || LLVM_DEBUG_EVERYTHING
-let public LLVM_JIT_ALLOW_UNALIGNED_VECTOR_READ_OUT_OF_BOUNDS_DEFAULT = false // NOTE - this should be true, to match interpreter and AOT behavior
-var public LLVM_JIT_ALLOW_UNALIGNED_VECTOR_READ_OUT_OF_BOUNDS = LLVM_JIT_ALLOW_UNALIGNED_VECTOR_READ_OUT_OF_BOUNDS_DEFAULT
-
let public LLVM_ENABLE_OPT_PASS = LLVM_DEBUG_EVERYTHING ? false : true
let public LLVM_LOG_RESULT = LLVM_DEBUG_EVERYTHING || LLVM_DEBUG_RESULT
let public LLVM_INLINE_THRESHOLD = 1024u // 225u
@@ -223,29 +221,6 @@ enum public JIT_SEQUENCE {
ITERATOR
}
-// We use global variable to store pointer to jit table methods.
-def public FN_JIT_TABLE_AT(t : Type) {
- return "`jit`table_at`{t}`"
-}
-
-def public FN_JIT_TABLE_ERASE(t : Type) {
- return "`jit`table_erase`{t}`"
-}
-
-def public FN_JIT_TABLE_FIND(t : Type) {
- return "`jit`table_find`{t}`"
-}
-
-// String-key at that takes a precomputed hash (string-only, no per-type matrix; the grow fallback
-// for the inline large-string at). String find is fully inline — no C++ helper.
-def public FN_JIT_STRING_TABLE_AT_WITH_HASH() {
- return "`jit`string_table_at_with_hash`"
-}
-
-def public FN_JIT_STRING_TABLE_AT_AFTER_PACKED_MISS() {
- return "`jit`string_table_at_after_packed_miss`"
-}
-
// nolint:STYLE014
// Address-carrying globals whose NAME is shared across split partitions (interp-call / interop
@@ -1035,7 +1010,6 @@ def public with_default_target_machine(opt_level : uint; use_host_cpu : bool;
// Offline AOT: emit a native object only, no linker step (jit_emit_object path).
// use_host_cpu=false (default) targets a generic CPU so the .o is portable; true bakes host features.
-def public jit_sec(us : int) : double => double(us) / 1000000.0lf
def public emit_object_only(mod : LLVMOpaqueModule?; out_path : string; use_host_cpu : bool = false) {
// Codegen level 3 deliberate: shipped artifacts, no cache guard (ARCHITECTURE.md 1.2)
diff --git a/modules/dasLLVM/daslib/llvm_jit_link.das b/modules/dasLLVM/daslib/llvm_jit_link.das
new file mode 100644
index 0000000000..af7994fc96
--- /dev/null
+++ b/modules/dasLLVM/daslib/llvm_jit_link.das
@@ -0,0 +1,133 @@
+options gen2
+options indenting = 4
+options strict_smart_pointers = false
+options stack = 4_194_304
+
+module llvm_jit_link shared private
+
+//! The JIT's entry, and the whole of a DLL cache hit. `just_in_time` requires this module and
+//! nothing of the emitter: the `jit_llvm` simulate macro computes the plan, opens the
+//! content-addressed DLL and binds the functions it holds. A miss, and every artifact but a
+//! cached DLL - an exe, a wasm, an AOT object, compile-only, an in-memory run, a pinned output
+//! path - brings the emitter's macro module in through `require_module_now` and runs `run_jit`
+//! there, so a warm run never loads the emitter's contexts.
+
+require llvm/daslib/llvm_jit_plan
+require daslib/ast_boost
+require daslib/cross_context
+require jit
+
+let private EMITTER_MODULE = "llvm/daslib/llvm_macro"
+
+[simulate_macro(name="jit_llvm")]
+class JIT_LLVM : AstSimulateMacro {
+ def override simulate(prog : Program?; var ctx : Context?) : bool {
+ return true if (is_in_completion() || is_compiling_macros() || is_folding())
+ return run_jit_linked(prog, ctx)
+ }
+}
+
+//! the link side's plan, its probe and the emitter's load are JIT work the emitter's own report
+//! never covers, so the miss path prints them on one line
+def private run_jit_in_emitter(prog : Program?; var ctx : Context?; log_jit_time : bool; t_plan, t_probe : int) : bool {
+ let load_tm = ref_time_ticks()
+ let mctx = macro_context_of(EMITTER_MODULE)
+ if (mctx == null) {
+ panic("LLVM JIT: the emitter module {EMITTER_MODULE} did not load")
+ }
+ if (log_jit_time) {
+ to_log(LOG_INFO, "LLVM JIT time: plan {jit_sec(t_plan)} probe {jit_sec(t_probe)} emitter load {jit_sec(get_time_usec(load_tm))}\n")
+ }
+ var ok = false
+ unsafe {
+ invoke_in_context(mctx, "run_jit_codegen", prog, ctx, addr(ok))
+ }
+ return ok
+}
+
+def private drop_probe(var dll : DLLHandle?; var uids : UidNodes?) {
+ dll.reset()
+ unsafe {
+ delete dll
+ delete uids
+ }
+}
+
+//! the hit path: plan, probe the DLL the plan names, bind what it holds; anything else is the
+//! emitter's. Returns true - a miss is codegen, not a failure.
+[macro_function]
+def public run_jit_linked(prog : Program?; var ctx : Context?) : bool {
+ if (prog.policies.aot && !prog.policies.jit_emit_object && !prog.policies.jit_enabled) {
+ return true
+ }
+ let totalTime = ref_time_ticks()
+ var phase_tm = ref_time_ticks()
+ var plan <- make_jit_plan(prog, ctx, true)
+ let t_hash = get_time_usec(phase_tm)
+ if (!plan.use_dll) {
+ return run_jit_in_emitter(prog, ctx, plan.log_jit_time, t_hash, 0)
+ }
+ if (empty(plan.candidates)) {
+ to_log(LOG_INFO, "LLVM JIT: 0 functions to jit ({length(plan.disabled)} marked no_jit) - the WHOLE program runs interpreted\n")
+ return true
+ }
+ phase_tm = ref_time_ticks()
+ var uids = create_uid_nodes(prog, plan.candidates, false)
+ var dll = get_dll_by_path(plan.output_path)
+ var probe <- probe_dll(plan.candidates, plan.disabled, dll, uids)
+ let t_probe = get_time_usec(phase_tm)
+ // a DLL holding none of the candidates is not a cache of this program: a pinned
+ // `-output` path another program wrote
+ if (dll.handle == null || probe.mismatched != 0 || empty(probe.bound)) {
+ drop_probe(dll, uids)
+ return run_jit_in_emitter(prog, ctx, plan.log_jit_time, t_hash, t_probe)
+ }
+ phase_tm = ref_time_ticks()
+ mark_jit_selection(prog, probe.bound)
+ let errors <- resolve_dll_externs(prog, dll, ctx, uids, probe.bound)
+ if (!empty(errors)) {
+ // a process fact the DLL name does not fold - a builtin with no interop address here, a
+ // handled type's helper from a module this process lacks: the emitter's content gate
+ // decides again and rewrites the DLL, where a panic would repeat on every run
+ for (e in errors) {
+ to_log(LOG_WARNING, "LLVM JIT: {e}\n")
+ }
+ to_log(LOG_WARNING, "LLVM JIT: the cached {plan.output_path}.dll cannot be linked into this process - {length(errors)} extern(s) unresolved, codegen\n")
+ drop_probe(dll, uids)
+ return run_jit_in_emitter(prog, ctx, plan.log_jit_time, t_hash, t_probe)
+ }
+ to_log(LOG_INFO, "LLVM JIT: DLL cache hit {plan.output_path}.dll\n")
+ install_dll_functions(dll, ctx, uids, probe.bound)
+ let t_install = get_time_usec(phase_tm)
+ phase_tm = ref_time_ticks()
+ set_jit_state(*ctx, dll.handle, null, null)
+ unsafe {
+ delete uids
+ }
+ let t_finalize = get_time_usec(phase_tm)
+ jit_log_report(probe.bound, false, plan.opt_level, !empty(plan.dll_hash_basename), plan.jit_split, 0, plan.debug_info, plan.log_jit_time, totalTime,
+ t_hash, 0, 0, t_probe, 0, 0, 0, t_install, t_finalize)
+ return true
+}
+
+//! Frees the jit state when the last context sharing it goes: the cached DLL closes here; an
+//! in-memory engine is the emitter's to dispose, and the emitter is in the process whenever
+//! there is one. One function, so a jitted program carries one finalizer and nothing else.
+[finalize(late), no_jit]
+def public free_jit_context {
+ if (get_context_share_counter() == 1ul) {
+ get_jit_state() $(dll, ee, ctx : void?) {
+ if (dll != null) {
+ close_dynamic_library(dll)
+ } elif (ee != null || ctx != null) {
+ let emitter = get_module("llvm_macro")
+ if (emitter == null || !has_macro_context(emitter)) {
+ panic("LLVM JIT: an in-memory engine exists but the emitter module is not in the process")
+ }
+ unsafe {
+ invoke_in_context(find_macro_context(emitter), "free_jit_engine", ee, ctx)
+ }
+ }
+ }
+ }
+}
diff --git a/modules/dasLLVM/daslib/llvm_jit_plan.das b/modules/dasLLVM/daslib/llvm_jit_plan.das
new file mode 100644
index 0000000000..99cce61320
--- /dev/null
+++ b/modules/dasLLVM/daslib/llvm_jit_plan.das
@@ -0,0 +1,913 @@
+options gen2
+options indenting = 4
+options no_unused_block_arguments = false
+options no_unused_function_arguments = false
+options strict_smart_pointers = false
+options relaxed_pointer_const
+options stack = 4_194_304
+options no_global_variables = false
+
+module llvm_jit_plan shared private
+
+//! The half of the JIT driver a cache hit needs, with no emitter behind it: the option
+//! resolution, the candidate set, the DLL key, the DLL probe and the install phase.
+//! `llvm_jit_link` runs a hit from here alone and brings the emitter (`llvm_jit_run`) in for
+//! a miss; the emitter builds on the same plan and the same install. The three host queries
+//! the key folds - triple, CPU name, CPU features - are the one use of the LLVM bindings here.
+
+require llvm/bindings/llvm_func
+require llvm/daslib/llvm_dll_utils public
+require llvm/daslib/llvm_env
+require llvm/daslib/llvm_cpu_class
+require llvm/daslib/llvm_jit_cli
+require daslib/ast_boost
+require daslib/strings_boost
+require daslib/fio
+require daslib/option
+require daslib/result
+require jit
+
+// Bump this constant whenever LLVM codegen / linking logic changes in a way that
+// invalidates cached DLLs (e.g. edits to llvm_jit.das, llvm_macro.das, llvm_jit_common.das,
+// runtime helper ABI, default target triple). Cache filenames fold this in, so a bump
+// makes every previously written DLL miss the cache on the next run and get GC'd.
+let public LLVM_JIT_CODEGEN_VERSION : uint64 = 0x7dul // 0x7d: the DLL key folds the candidate set - every used function the policy selects, before the emitter's content gate - so a cache hit binds what the DLL holds with no emitter loaded (0x7c: under --jit-debug a das global is described too, at the ctx-derived pointer each function computes for it; every das local, argument and block variable carries a DILocalVariable with a real DIType, a das block opens a DILexicalBlock, a subprogram is named as the das source names it and carries no linkage name, one compile unit covers the whole module, and the prologue and a loop latch carry no stale line (0x7a: a lambda <-> int64/uint64 reinterpret lowers through ptrtoint / inttoptr, not a bitcast (0x79: an intrinsic emitter keys off the declared name, so a module-qualified call (math::sqrt) builds llvm.sqrt.f32, and a function <-> int64/uint64 reinterpret extracts and inserts the { ptr } aggregate instead of bitcasting it (0x78: a standalone exe adopts its emitter-sealed function and global lookups right after the context is created (0x77: a vector of a handled element type registers into the element's module, so the externs a DLL binds by mangled name moved out of `$` (0x76: a statement after a terminator in the same block list lands in its own dead block instead of after the ret (0x75: the global-offset lookup is memory(none) and emitted at its use site - LLVM dedups and hoists it, an untaken branch never pays it; a solid-context global resolves once per function at entry (0x74: a runtime-only exe emits no register_native_path rows, and a whole-lib exe emits them once (0x73: computed goto lowers to one switch with the trap as its default, not an icmp chain (0x72: policies.fast_math defaults to the host's float flags, so a fast-math host now JITs fast-math (0x71: a CPU class row's cpu is the arch's bare baseline, so a DAS_JIT_BASELINE build enables the row's set and nothing a level implies (0x70: the wasm feature string drops +relaxed-simd and the idot family keeps only the exact extmul + extadd_pairwise lowering on wasm SIMD128 (0x6f: the first wasm idot lowering; 0x6e: the aarch64 SDOT / SMMLA tables gate on DotProd / i8mm, not the arch alone, and the force env reaches the generic exe machine (0x6d: the inline polynomial rail carries NaN: tanh selects the operand back over its ordered clamp, and the sincos quadrant / tan octant convert through llvm.fptosi.sat instead of poisoning on NaN and out-of-range (0x6c: aarch64 vector tan/exp2/log2/log/pow join the inline polynomial rail bit-exactly with the interpreter, sinh/cosh/tanh ride the exp one; 0x6b: aarch64 vector sin/cos ride the inline polynomial; 0x6a: srem/urem for 32-bit %; 0x69: every string argument of an extern is substituted, not just the ones which asked))))
+
+let public JIT_FNV_PRIME : uint64 = 1099511628211ul
+
+// `options fast_math` stamps every FP op with VALUE-SAFE flags (reassoc|nsz|contract) before opt —
+// NON-bit-exact, default off, +8-10% e2e. nnan|ninf|arcp|afn are deliberately NOT stamped: engine
+// kernels make real infs and arcp+ninf's rcp-estimate div computes inf*0=NaN (tests/jit/fast_math_specials.das).
+var public g_jit_fast_math = false
+
+let public LLVM_JIT_ALLOW_UNALIGNED_VECTOR_READ_OUT_OF_BOUNDS_DEFAULT = false // NOTE - this should be true, to match interpreter and AOT behavior
+var public LLVM_JIT_ALLOW_UNALIGNED_VECTOR_READ_OUT_OF_BOUNDS = LLVM_JIT_ALLOW_UNALIGNED_VECTOR_READ_OUT_OF_BOUNDS_DEFAULT
+
+// We use global variable to store pointer to jit table methods.
+def public FN_JIT_TABLE_AT(t : Type) {
+ return "`jit`table_at`{t}`"
+}
+
+def public FN_JIT_TABLE_ERASE(t : Type) {
+ return "`jit`table_erase`{t}`"
+}
+
+def public FN_JIT_TABLE_FIND(t : Type) {
+ return "`jit`table_find`{t}`"
+}
+
+// String-key at that takes a precomputed hash (string-only, no per-type matrix; the grow fallback
+// for the inline large-string at). String find is fully inline — no C++ helper.
+def public FN_JIT_STRING_TABLE_AT_WITH_HASH() {
+ return "`jit`string_table_at_with_hash`"
+}
+
+def public FN_JIT_STRING_TABLE_AT_AFTER_PACKED_MISS() {
+ return "`jit`string_table_at_after_packed_miss`"
+}
+
+def public jit_sec(us : int) : double => double(us) / 1000000.0lf
+
+// nolint:STYLE014
+// Loop tune hints (vectorize / vectorize_width / unroll / unroll_count / interleave_count)
+// live on ExprFor / ExprWhile annotations and reach ONLY the JIT — attach_loop_metadata
+// turns them into LLVM !llvm.loop metadata. They never enter the interpreter SimNode tree
+// that get_function_aot_hash walks, so the semantic hash is blind to them: a re-tuned
+// kernel ([tuned] restamping a different perm from a changed box_profile.json) has an
+// IDENTICAL aot hash and would silently reuse a stale DLL. Fold them into the JIT-only
+// cache key here, alongside LLVM_JIT_CODEGEN_VERSION / g_jit_fast_math.
+// Fold the ONE value field the arg's type says is live. bValue/iValue/fValue share a C++
+// union — reading a non-live member is UB (a bool load of vectorize_width=8 aborts ubsan
+// and lets clang -O2 miscompile: chunk workers died with SIGSEGV on linux Release CI).
+// Type coverage stays total: any basicType re-keys the DLL via its own live field.
+def public fold_annotation_args(args : AnnotationArgumentList; seed : uint64) : uint64 {
+ var h = seed
+ for (a in args) {
+ h = (h ^ hash(a.name)) * JIT_FNV_PRIME
+ h = (h ^ uint64(int(a.basicType))) * JIT_FNV_PRIME
+ if (a.basicType == Type.tBool) {
+ h = (h ^ (a.bValue ? 1ul : 0ul)) * JIT_FNV_PRIME
+ } elif (a.basicType == Type.tInt) {
+ h = (h ^ uint64(a.iValue)) * JIT_FNV_PRIME
+ } elif (a.basicType == Type.tFloat) {
+ h = (h ^ hash(a.fValue)) * JIT_FNV_PRIME
+ } elif (a.basicType == Type.tString) {
+ h = (h ^ hash(a.sValue)) * JIT_FNV_PRIME
+ }
+ }
+ return h
+}
+
+class private LoopHintHashVisitor : AstVisitor {
+ h : uint64
+ def override preVisitExprFor(expr : ExprFor?) : void {
+ h = fold_annotation_args(expr.annotations, h)
+ }
+ def override preVisitExprWhile(expr : ExprWhile?) : void {
+ h = fold_annotation_args(expr.annotations, h)
+ }
+}
+
+// Complete under jit_all_functions (the default): every used function is a top-level entry
+// in `funcs`, so folding each function's own body covers the whole compiled program.
+def public fold_loop_hints(funcs : array; seed : uint64) : uint64 {
+ var vis = new LoopHintHashVisitor()
+ vis.h = seed
+ make_visitor(*vis) $(adapter) {
+ for (fn in funcs) {
+ visit(fn.body, adapter)
+ }
+ }
+ let r = vis.h
+ unsafe { delete vis }
+ return r
+}
+
+// [llvm_code] annotation args are generator params: they change emitted IR without touching the
+// reference body, so the aot hash is blind — fold them like loop hints. A generator CODE change is
+// invisible too: bump LLVM_JIT_CODEGEN_VERSION when a generator's emission changes for fixed args.
+def public fold_llvm_code_hints(funcs : array; seed : uint64) : uint64 {
+ var h = seed
+ for (fn in funcs) {
+ for (ann in fn.annotations) {
+ if (ann.annotation.name == "llvm_code") {
+ h = (h ^ hash("llvm_code")) * JIT_FNV_PRIME
+ h = fold_annotation_args(ann.arguments, h)
+ }
+ }
+ }
+ return h
+}
+
+// [hint(...)] args become LLVM function attributes (alwaysinline/noalias=/optnone/...) — backend-only,
+// no SimNode, so the aot hash is blind to them exactly like loop hints; fold them into both keys
+def public fold_function_hints(funcs : array; seed : uint64) : uint64 {
+ var h = seed
+ for (fn in funcs) {
+ for (ann in fn.annotations) {
+ if (ann.annotation.name == "hint") {
+ h = (h ^ hash("hint")) * JIT_FNV_PRIME
+ h = fold_annotation_args(ann.arguments, h)
+ }
+ }
+ }
+ return h
+}
+
+// True when the program jits any [llvm_code] kernel — those emit host-specific IR (e.g. AVX2) a
+// generic x86-64 exe can't legalize, so a -exe carrying them must target the current box. Scanned
+// BEFORE target flags: the generators' decline predicates consult g_target_*, so order matters.
+def private has_generated_kernel(prog : Program?; jit_all_functions : bool) : bool {
+ var found = false
+ prog |> for_each_module() $(mod) {
+ mod |> for_each_function("") $(fun) {
+ if (is_used(prog, fun) && (jit_all_functions || fun.moreFlags.requestJit) && !fun.moreFlags.requestNoJit) {
+ for (ann in fun.annotations) {
+ if (ann.annotation.name == "llvm_code") {
+ found = true
+ }
+ }
+ }
+ }
+ }
+ return found
+}
+
+// The environment/config salt shared by the whole-DLL key AND the per-partition obj cache key.
+// ONE helper on purpose: a salt component folded into the DLL key but not the partition key
+// would let a config change reuse stale cached partition objects.
+def public jit_env_salt(opt_level : int; size_level : int; emit_prologue : bool; debug_info : bool; // nolint:STYLE037,STYLE038 - one flat fold per emission input; a split would hide which inputs key the cache
+ target_triple : string; split : bool; use_host_cpu : bool) : uint64 {
+ var h = LLVM_JIT_CODEGEN_VERSION
+ // host/binary identity: object format follows the triple (COFF vs ELF on one checkout), and the emitting binary bakes its Context offsets into every global access - neither is in any AST hash
+ if (empty(target_triple)) {
+ let host_triple = LLVMGetDefaultTargetTriple()
+ h = (h ^ hash(host_triple)) * JIT_FNV_PRIME
+ LLVMDisposeMessage(host_triple)
+ }
+ h = (h ^ uint64(CONTEXT_OFFSET_OF_GLOBALS)) * JIT_FNV_PRIME
+ h = (h ^ uint64(CONTEXT_OFFSET_OF_SHARED)) * JIT_FNV_PRIME
+ h = (h ^ uint64(CONTEXT_OFFSET_OF_STOP_FLAGS)) * JIT_FNV_PRIME
+ h = (h ^ uint64(CONTEXT_OFFSET_OF_EVAL_TOP)) * JIT_FNV_PRIME
+ // full host-CPU identity when the TargetMachine bakes it - the cpuid bits below cover only the matrix-tier features; a wider box's object is an illegal instruction on a narrower one
+ if (use_host_cpu) {
+ let host_cpu = LLVMGetHostCPUName()
+ let host_features = LLVMGetHostCPUFeatures()
+ h = (h ^ hash(host_cpu)) * JIT_FNV_PRIME
+ h = (h ^ hash(host_features)) * JIT_FNV_PRIME
+ LLVMDisposeMessage(host_cpu)
+ LLVMDisposeMessage(host_features)
+ }
+ // DAS_JIT_PROBE_LTO artifacts (bitcode + lld-LTO link) must never serve a normal run
+ if (g_env_jit.jit_probe_lto) {
+ h = (h ^ hash("lto_probe")) * JIT_FNV_PRIME
+ }
+ // options jit_vec3_ldu selects the vector-load emission shape
+ h = (h ^ (LLVM_JIT_ALLOW_UNALIGNED_VECTOR_READ_OUT_OF_BOUNDS ? 1ul : 0ul)) * JIT_FNV_PRIME
+ h = (h ^ uint64(opt_level)) * JIT_FNV_PRIME
+ h = (h ^ uint64(size_level)) * JIT_FNV_PRIME
+ h = (h ^ (emit_prologue ? 1ul : 0ul)) * JIT_FNV_PRIME
+ h = (h ^ (debug_info ? 1ul : 0ul)) * JIT_FNV_PRIME
+ h = (h ^ (g_jit_fast_math ? 1ul : 0ul)) * JIT_FNV_PRIME // fast-math changes codegen → distinct cache
+ // split-built and monolith DLLs differ in inlining — never share a cache entry (bool only: partition count is data-driven)
+ h = (h ^ (split ? 1ul : 0ul)) * JIT_FNV_PRIME
+ // EH vs longjmp hosts emit different unwind tables — must not share a cache entry (panic would crash)
+ h = (h ^ (das_is_exceptions_enabled() ? 1ul : 0ul)) * JIT_FNV_PRIME
+ // AVX2 gates the x64 intrinsic table — an AVX2-jitted DLL must not cache-hit an AVX2-less host (illegal instr)
+ h = (h ^ (cpu_supports("avx2") ? 1ul : 0ul)) * JIT_FNV_PRIME
+ // Matrix-tier cpuid bits (vnni256/avx512bw/avx512vnni) + DAS_JIT_X64_FORCE_FEATURES key emission the same way
+ var isa = 0ul
+ if (cpu_supports("avxvnni")) {
+ isa |= 1ul
+ }
+ if (cpu_supports("avx512f")) {
+ isa |= 2ul
+ }
+ if (cpu_supports("avx512bw")) {
+ isa |= 4ul
+ }
+ if (cpu_supports("avx512vl")) {
+ isa |= 8ul
+ }
+ if (cpu_supports("avx512vnni")) {
+ isa |= 16ul
+ }
+ if (cpu_supports("avx512vbmi")) { // the grid formats' symbol lattice emits on it (grid = "vbmi")
+ h = (h ^ 0x1Bul) * JIT_FNV_PRIME
+ }
+ if (cpu_supports("amx-tile") && cpu_supports("amx-int8")) { // tune_cpu_class picks x86-amx, which selects grid decode forms
+ h = (h ^ 0x1Cul) * JIT_FNV_PRIME
+ }
+ if (cpu_supports("avxvnniint8")) {
+ isa |= 32ul
+ }
+ h = (h ^ isa) * JIT_FNV_PRIME
+ let forced = g_env_jit.jit_x64_force_features
+ if (!empty(forced)) {
+ h = (h ^ hash(forced)) * JIT_FNV_PRIME
+ }
+ // aarch64 twin (DAS_JIT_ARM64_FORCE_FEATURES, e.g. "i8mm"): a forced artifact must not cache-hit a normal run
+ let forced_arm = g_env_jit.jit_arm64_force_features
+ if (!empty(forced_arm)) {
+ h = (h ^ hash(forced_arm)) * JIT_FNV_PRIME
+ }
+ // aarch64 i8mm is an emission input (the +i8mm append): a warm pre-i8mm cache must not serve an i8mm host
+ h = (h ^ (cpu_supports("i8mm") ? 64ul : 0ul)) * JIT_FNV_PRIME
+ // a baseline class (DAS_JIT_BASELINE) picks the machine, the gates and the tune ladder - its artifact never serves a host run
+ let baseline = jit_baseline_class()
+ if (!empty(baseline)) {
+ h = (h ^ hash(baseline)) * JIT_FNV_PRIME
+ }
+ // Cross-compile artifacts key on their triple — a --jit-target DLL must not cache-hit a host run (foreign-arch dlopen)
+ if (!empty(target_triple)) {
+ h = (h ^ hash(target_triple)) * JIT_FNV_PRIME
+ }
+ return h
+}
+
+def public jit_dll_basename(funcs : array; opt_level : int; size_level : int;
+ emit_prologue : bool; debug_info : bool; target_triple : string; split : bool;
+ use_host_cpu : bool) : string {
+ var h = LLVM_JIT_CODEGEN_VERSION
+ for (fn in funcs) {
+ h = (h ^ get_function_aot_hash(fn)) * JIT_FNV_PRIME
+ }
+ h = fold_loop_hints(funcs, h) // JIT-only loop tune hints (see LoopHintHashVisitor)
+ h = fold_llvm_code_hints(funcs, h) // [llvm_code] generator params (annotation args)
+ h = fold_function_hints(funcs, h) // [hint(...)] function attributes (backend-only, aot-hash-blind)
+ h = (h ^ jit_env_salt(opt_level, size_level, emit_prologue, debug_info, target_triple, split, use_host_cpu)) * JIT_FNV_PRIME
+ return "{h}"
+}
+
+def public jit_gc_stale_dlls(subdir : string; keep_basename : string; keep_names : table) {
+ // keep mask: the current DLL-hash prefix (dll/lib/exp/pdb/rsp + monolith .o) OR an exact keep_names entry (split obj cache's live partition objects - named by partition key, not dll hash)
+ let keep_prefix = "{keep_basename}."
+ dir(subdir) $(fname) {
+ if (fname == "." || fname == ".." || fname |> starts_with(keep_prefix) || keep_names |> key_exists(fname)) return
+ let full_path = "{subdir}/{fname}"
+ var err : string
+ let ok = remove(full_path, err)
+ if (ok && empty(err)) {
+ to_log(LOG_INFO, "LLVM JIT: GC stale artifact {full_path}\n")
+ } else {
+ to_log(LOG_WARNING, "LLVM JIT: failed to GC stale artifact {full_path}: {err}\n")
+ }
+ }
+}
+
+struct public JitState {
+ dll : void?
+ ee : void?
+ ctx : void?
+}
+
+// always-on summary (a silent multi-second jit is indistinguishable from a hang) + the
+// log_compile_time per-phase breakdown; both artifact paths report through here
+def public jit_log_report(funcs : array; recompiled : bool; opt_level : int; tier_known : bool;
+ split : bool; split_parts : int; debug_info : bool; log_time : bool; totalTime : int64;
+ t_hash, t_init, t_declare, t_probe, t_irgen, t_opt, t_emit, t_install, t_finalize : int) {
+ var tuneStamped = 0
+ for (fn in funcs) {
+ for (ann in fn.annotations) {
+ if (ann.annotation.name == "llvm_code" && find_arg(ann.arguments, "tune_suffix") is tString) {
+ tuneStamped++
+ break
+ }
+ }
+ }
+ let tuneTail = tuneStamped > 0 ? " incl {tuneStamped} tune-stamped" : ""
+ let tierTail = tier_known ? ", O{opt_level}" : ""
+ // always announce split - a split-built DLL's profile is not comparable to a monolith build
+ let splitTail = !split ? "" : (split_parts > 0 ? ", split {split_parts} modules" : ", split")
+ let debugTail = debug_info ? ", --jit-debug" : ""
+ let totalDt = double(get_time_usec(totalTime)) / 1000000.0lf
+ to_log(LOG_INFO, "LLVM JIT: {length(funcs)} functions{tuneTail} in {totalDt} sec ({recompiled ? "codegen" : "cache"}{tierTail}{splitTail}{debugTail})\n")
+ if (log_time) {
+ to_log(LOG_INFO, "LLVM JIT time: hash {jit_sec(t_hash)} init {jit_sec(t_init)} declare {jit_sec(t_declare)} probe {jit_sec(t_probe)} irgen {jit_sec(t_irgen)} optimize {jit_sec(t_opt)} emit+link {jit_sec(t_emit)} install {jit_sec(t_install)} finalize {jit_sec(t_finalize)}\n")
+ }
+}
+
+// Check if a function is a late-bound dasbind function (null builtin address, resolvable at runtime)
+def public is_dasbind_late(func : Function?) : bool {
+ if (func == null || !func.flags.builtIn || func._module.name != "dasbind") return false
+ var hit = false
+ peek(func.name) $(s) {
+ hit = starts_with(s, "__dasbind__")
+ }
+ return hit
+}
+
+def public handle_each_is_jitable(src : ExpressionPtr) : bool {
+ assume srcT = src._type
+ if (srcT == null || !srcT.isHandle || srcT.annotation == null) {
+ return false
+ }
+ return get_jit_each(srcT.annotation) != null
+}
+
+//! The jit passes a handled index in its own C ABI, so the index has to be a primitive the
+//! emitter can put in a register. Anything else (a typed id, a struct key) gets no address and
+//! keeps the interpreter, whatever the annotation offers. Returns -1 when not passable.
+def public jit_index_base_type(ixT : TypeDeclPtr) : int {
+ if (ixT == null || ixT.isRef) {
+ return -1
+ }
+ let bt = ixT.baseType
+ if (bt == Type.tInt || bt == Type.tUInt || bt == Type.tInt64 || bt == Type.tUInt64 ||
+ bt == Type.tInt8 || bt == Type.tUInt8 || bt == Type.tInt16 || bt == Type.tUInt16 ||
+ bt == Type.tFloat || bt == Type.tString) {
+ return int(bt)
+ }
+ return -1
+}
+
+def public handle_at_is_jitable(expr : ExprAt?) : bool {
+ assume subExprT = expr.subexpr._type
+ if (!subExprT.isHandle || subExprT.annotation == null) {
+ return false
+ }
+ let ibt = jit_index_base_type(expr.index._type)
+ if (ibt < 0) {
+ return false
+ }
+ return get_jit_at(subExprT.annotation, ibt) != null
+}
+
+def public loop_source_is_jitable(src : ExpressionPtr) : bool {
+ assume srcT = src._type
+ if (srcT == null) {
+ return false
+ }
+ return (srcT.isRange || srcT.baseType == Type.tFixedArray || srcT.isGoodArrayType
+ || srcT.isIterator || srcT.isString || handle_each_is_jitable(src))
+}
+
+//! the interpreter's address of `func` in `ctx`; null when the context holds no such function
+def public interpreter_function_addr(var ctx : Context?; func : Function?) : void? {
+ let MNH = hash(get_mangled_name(func))
+ let MNH_ADDR = get_function_address(MNH, *ctx)
+ return MNH_ADDR == 0ul ? null : unsafe(reinterpret(MNH_ADDR))
+}
+
+def public get_dll_missing(fns : array; disabled : array; var dll : DLLHandle?; var uids : UidNodes?) {
+ verify(!(is_in_completion() || is_compiling_macros() || is_folding()))
+
+ var missing = array()
+ missing |> reserve(length(fns))
+ var disabled_exists = array()
+ if (dll.handle == null) {
+ for (fn in fns) {
+ missing |> push(unsafe(reinterpret(fn)))
+ }
+ } else {
+ for (fn in fns) {
+ let dll_name = uids.get_dll_fn_name(fn)
+ var written_hash = dll.get_function_hash(dll_name)
+ if (written_hash == null || get_function_aot_hash(fn) != *written_hash) {
+ missing |> push(unsafe(reinterpret(fn)))
+ }
+ }
+ for (fn in disabled) {
+ let dll_name = uids.get_dll_fn_name(fn)
+ var written_hash = dll.get_function_hash(dll_name)
+ if (written_hash != null) {
+ disabled_exists |> push(unsafe(reinterpret(fn)))
+ }
+ }
+ }
+ return <- (missing, disabled_exists)
+}
+
+//! the candidate set against a DLL: a candidate the DLL holds under its own hash is bound; one
+//! the DLL lacks was dropped by the emitter's content gate and stays interpreted; a candidate
+//! held under another hash, or a `[no_jit]` function the DLL holds, is a miss
+def public probe_dll(var candidates : array; disabled : array; var dll : DLLHandle?; var uids : UidNodes?) {
+ verify(!(is_in_completion() || is_compiling_macros() || is_folding()))
+ var bound : array
+ var mismatched = 0
+ if (dll.handle != null) {
+ for (fn in candidates) {
+ var written_hash = dll.get_function_hash(uids.get_dll_fn_name(fn))
+ if (written_hash == null) {
+ continue
+ }
+ if (get_function_aot_hash(fn) != *written_hash) {
+ mismatched++
+ } else {
+ bound |> push(fn)
+ }
+ }
+ for (fn in disabled) {
+ if (dll.get_function_hash(uids.get_dll_fn_name(fn)) != null) {
+ mismatched++
+ }
+ }
+ } else {
+ mismatched = length(candidates)
+ }
+ return <- (bound <- bound, mismatched = mismatched)
+}
+
+//! the program's jit selection becomes `selected` and nothing else - the emitter and the extern
+//! resolver read it to tell a call into the DLL from a call through a slot
+def public mark_jit_selection(prog : Program?; var selected : array) {
+ clear_jit_selection(prog)
+ for (fn in selected) {
+ set_jit_selected(prog, fn, true)
+ }
+}
+
+//! Fills the DLL's extern slots for the functions it holds - interop addresses of the builtins
+//! called, interpreter addresses of the callees the DLL lacks, the handled-type and table helpers.
+//! A slot the DLL lacks was lowered inline and is skipped; one nothing can fill lands in `errors`.
+class public DllExternResolver : AstVisitor {
+ @do_not_delete prog : Program?
+ @do_not_delete dll : DLLHandle?
+ @do_not_delete jit_context : Context?
+ @do_not_delete thisFunc : Function?
+ @do_not_delete thisBlock : array
+ @do_not_delete uid : UidNodes?
+ errors : array
+
+ def override canVisitWithAliasSubexpression(expr : ExprAssume?) : bool {
+ return false
+ }
+
+ def override preVisitFunction(var fun : FunctionPtr) : void {
+ uid.reset(fun)
+ thisFunc = fun
+ }
+
+ def override visitFunction(fun : FunctionPtr) : FunctionPtr {
+ thisFunc = null
+ uid.reset(null)
+ return fun
+ }
+
+ def DllExternResolver(_prog : Program?; var _dll : DLLHandle?; ctx : Context?; uids : UidNodes?) {
+ prog = _prog
+ dll = _dll
+ jit_context = ctx
+ uid = uids
+ }
+
+ def private fail(msg : string) {
+ errors |> push(msg)
+ }
+
+ //! `what` names the slot only on failure - the message is a `describe` the success path never pays for
+ def private set_slot(name : DllName; ptr : void?; what : block<() : string>) {
+ if (dll.set_glob_address(name, ptr) && ptr == null) {
+ fail("missing {invoke(what)}")
+ }
+ }
+
+ def make_call(expr : ExprCallFunc?) {
+ if (expr.func == null) return
+ let slot = uid.get_dll_fn_name_ptr(expr.func)
+ if (expr.func.flags.builtIn) {
+ let FUNC_PTR = get_builtin_function_address(expr.func)
+ if (dll.set_glob_address(slot, FUNC_PTR) && FUNC_PTR == null && !is_dasbind_late(expr.func)) {
+ fail("missing interop function pointer for {expr.func._module.name}::{expr.func.name} // {get_mangled_name(expr.func)} at {describe(expr.at)}")
+ }
+ } elif (!is_jit_selected(prog, expr.func)) {
+ set_slot(slot, interpreter_function_addr(jit_context, expr.func)) {
+ return "function pointer for {expr.func.name} at {describe(expr.at)}"
+ }
+ }
+ }
+
+ def override preVisitExprNew(expr : ExprNew?) {
+ if (expr.typeexpr.isHandle) {
+ set_slot(uid.get_new(expr.typeexpr), get_jit_new(expr.typeexpr.annotation)) {
+ return "new`handle function pointer for {describe(expr.typeexpr)}"
+ }
+ } elif (expr.typeexpr.baseType == Type.tFixedArray) {
+ // the handle gate lives on the chain element (the head is tFixedArray);
+ // borrow-walk — this pass runs outside ast_gc_guard, clones would leak
+ var elemT = expr.typeexpr
+ while (elemT.baseType == Type.tFixedArray && elemT.firstType != null) {
+ elemT = elemT.firstType
+ }
+ if (elemT.baseType == Type.tHandle) {
+ set_slot(uid.get_new(expr.typeexpr), get_jit_new(elemT.annotation)) {
+ return "new`handle function pointer for {describe(expr.typeexpr)}"
+ }
+ } elif (expr.initializer) {
+ make_call(expr)
+ }
+ } elif (expr.initializer) {
+ make_call(expr)
+ }
+ }
+
+ def override preVisitExprCall(expr : ExprCall?) {
+ make_call(expr)
+ }
+
+ def override preVisitExprOp2(var expr : ExprOp2?) {
+ if (expr.func != null) {
+ make_call(expr)
+ }
+ }
+
+ def override preVisitExprOp1(var expr : ExprOp1?) {
+ if (expr.func != null) {
+ make_call(expr)
+ }
+ }
+
+ def override preVisitExprDelete(var expr : ExprDelete?) {
+ assume subT = expr.subexpr._type
+ if (subT.isPointer && subT.firstType.isHandle) {
+ assume firstT = subT.firstType
+ set_slot(uid.get_delete_type(firstT), get_jit_delete(firstT.annotation)) {
+ return "delete`handle function pointer for {describe(firstT)}"
+ }
+ }
+ }
+
+ def override preVisitExprClone(expr : ExprClone?) : void {
+ assume leftT = expr.left._type
+ if (leftT.isHandle) {
+ set_slot(uid.get_clone_type(leftT), get_jit_clone(leftT.annotation)) {
+ return "clone`handle function pointer for {describe(leftT)}"
+ }
+ }
+ }
+
+ def override preVisitExprAddr(expr : ExprAddr?) {
+ set_slot(uid.get_addr(expr), interpreter_function_addr(jit_context, expr.func)) {
+ return "function pointer for {expr.func.name}"
+ }
+ }
+
+ def override preVisitExprMakeStructIndex(expr : ExprMakeStruct?; index : int; last : bool) : void {
+ if (expr.constructor != null && !is_jit_selected(prog, expr.constructor)) {
+ set_slot(uid.get_ctor(expr), interpreter_function_addr(jit_context, expr.constructor)) {
+ return "function pointer for {expr.constructor.name}"
+ }
+ }
+ }
+
+ def override preVisitExprAscend(expr : ExprAscend?) : void {
+ if (expr.subexpr._type.isHandle) {
+ assume firstT = expr.subexpr._type
+ set_slot(uid.get_ascend_new(expr), get_jit_new(firstT.annotation)) {
+ return "new`handle function pointer for {describe(firstT)}"
+ }
+ }
+ }
+
+ def override preVisitExprMakeBlock(expr : ExprMakeBlock?) : void {
+ thisBlock |> push <| unsafe(reinterpret(expr._block))
+ }
+
+ def override visitExprMakeBlock(expr : ExprMakeBlock?) : ExpressionPtr {
+ thisBlock |> pop()
+ return expr
+ }
+
+ def private get_table_t(subexprT : TypeDeclPtr) {
+ assume firstT = subexprT.firstType
+ if (firstT.isWorkhorseType) {
+ return firstT.baseType
+ } else {
+ var uvt = get_underlying_value_type(firstT)
+ return uvt.baseType
+ }
+ }
+
+ // Pre-bind the JIT table-fn globals for the standalone-exe / cache-reload path. String
+ // keys route through the precomputed-hash helpers (see build_table_find/at), so bind
+ // those globals instead of the per-type ones.
+ def bind_jit_table_find_glob(t : Type) {
+ // String find is fully inline (no glob). Non-string binds the per-type C++ find glob — still
+ // used by non-workhorse keys (vectors / structs) that route through build_table_find_call.
+ if (t != Type.tString) {
+ dll.set_glob_address(DllName(FN_JIT_TABLE_FIND(t)), get_jit_table_find(unsafe(reinterpret(t))))
+ }
+ }
+ def bind_jit_table_at_glob(t : Type) {
+ if (t == Type.tString) {
+ dll.set_glob_address(DllName(FN_JIT_STRING_TABLE_AT_WITH_HASH()), get_jit_string_table_at_with_hash())
+ dll.set_glob_address(DllName(FN_JIT_STRING_TABLE_AT_AFTER_PACKED_MISS()), get_jit_string_table_at_after_packed_miss())
+ } else {
+ // Non-string keys are open-addressed: the inline probe handles find/at hits and routes a
+ // miss to the regular C++ at (FN_JIT_TABLE_AT) — no packed after-miss helper needed.
+ dll.set_glob_address(DllName(FN_JIT_TABLE_AT(t)), get_jit_table_at(unsafe(reinterpret(t))))
+ }
+ }
+
+ def override preVisitExprAt(expr : ExprAt?) {
+ assume subExprT = expr.subexpr._type
+ if (subExprT.isGoodTableType) {
+ bind_jit_table_at_glob(get_table_t(subExprT))
+ } elif (handle_at_is_jitable(expr)) {
+ assume ixT = expr.index._type
+ dll.set_glob_address(uid.get_at_type(subExprT, ixT),
+ get_jit_at(subExprT.annotation, jit_index_base_type(ixT)))
+ }
+ }
+
+ def override preVisitExprForSource(var expr : ExprFor?, var source : ExpressionPtr, last : bool) : void {
+ if (handle_each_is_jitable(source)) {
+ dll.set_glob_address(uid.get_each_type(source._type), get_jit_each(source._type.annotation))
+ }
+ }
+
+ def override preVisitExprSetInsert(expr : ExprSetInsert?) {
+ bind_jit_table_at_glob(get_table_t(expr.arguments[0]._type))
+ }
+
+ def override preVisitExprKeyExists(expr : ExprKeyExists?) {
+ bind_jit_table_find_glob(get_table_t(expr.arguments[0]._type))
+ }
+
+ def override preVisitExprFind(expr : ExprFind?) {
+ bind_jit_table_find_glob(get_table_t(expr.arguments[0]._type))
+ }
+
+ def override preVisitExprSafeAt(expr : ExprSafeAt?) {
+ assume subExprT = expr.subexpr._type
+ if (subExprT.isGoodTableType || (subExprT.isPointer && subExprT.firstType.isGoodTableType)) {
+ var etype = subExprT.isPointer ? subExprT.firstType : subExprT
+ bind_jit_table_find_glob(get_table_t(etype))
+ }
+ }
+
+ def override preVisitExprErase(expr : ExprErase?) {
+ let t = get_table_t(expr.arguments[0]._type)
+ dll.set_glob_address(DllName(FN_JIT_TABLE_ERASE(t)), get_jit_table_erase(unsafe(reinterpret(t))))
+ }
+}
+
+//! fills the DLL's extern slots for `funcs`; the errors, empty on success
+def public resolve_dll_externs(prog : Program?; var dll : DLLHandle?; var ctx : Context?; var uids : UidNodes?; funcs : array) : array {
+ var resolver = new DllExternResolver(prog, dll, ctx, uids)
+ make_visitor(*resolver) $(adapter) {
+ for (fn in funcs) {
+ visit(fn, adapter)
+ }
+ }
+ var errors <- resolver.errors
+ unsafe {
+ delete resolver
+ }
+ return <- errors
+}
+
+//! points every function's sim node at its code in the DLL
+def public install_dll_functions(var dll : DLLHandle?; var ctx : Context?; var uids : UidNodes?; funcs : array) {
+ for (fun in funcs) {
+ let simfun = get_function_by_mangled_name_hash(hash(get_mangled_name(fun)), *ctx)
+ var gen = dll.get_function_address(uids.get_dll_fn_name(fun))
+ if (gen != null) {
+ gen |> unsafe(instrument_jit(simfun, fun.at, *ctx))
+ } else {
+ unsafe(remove_jit(simfun))
+ }
+ }
+}
+
+//! Everything `run_jit` decides before it touches LLVM: the resolved options, the candidate set
+//! (every used function the policy selects, `[no_jit]` aside) and the content-addressed DLL name.
+//! Its inputs are the program and the command line, so the link module and the emitter compute the same plan.
+struct public JitPlan {
+ aot_host : bool
+ jit_all_functions : bool
+ debug_info : bool
+ dump_ir : bool
+ gen_exe : bool
+ exe_main : string
+ compile_only : bool
+ emit_aot_object : bool
+ use_dll : bool
+ split_requested : bool
+ split_modules : int
+ jit_split : bool
+ obj_cache : bool
+ opt_level : int
+ size_level : int
+ emit_prologue : bool
+ exe_strict : bool
+ config_out_path : string
+ output_path : string
+ path_to_shared_lib : string
+ path_to_linker : string
+ target_triple : string
+ gen_wasm : bool
+ runtime_lib_override : string
+ emit_object : bool
+ wasm_threads : bool
+ check_abi : bool
+ linker_string : string
+ register_all_modules : bool
+ baseline : string
+ exe_host_cpu : bool
+ use_host_cpu : bool
+ log_jit_time : bool
+ dll_hash_basename : string
+ candidates : array
+ disabled : array
+}
+
+def private jit_selects(fun : FunctionPtr; rqj : bool; aot_host : bool; jit_all : bool; var ctx : Context?) : bool {
+ if (rqj) return true
+ if (!aot_host) return jit_all
+ return !is_aot_function(get_function_by_mangled_name_hash(hash(get_mangled_name(fun)), *ctx))
+}
+
+//! the plan for `prog`; `announce` prints the one-time lines (the baseline class, the exe host
+//! CPU) - the first computation in a run passes true, a recomputation false
+def public make_jit_plan(prog : Program?; var ctx : Context?; announce : bool) : JitPlan { // nolint:STYLE038 - one flat resolution of every emission input; a split would hide which inputs key the cache
+ var plan : JitPlan
+ plan.aot_host = prog.policies.aot && prog.policies.jit_enabled
+ let r <- parse_args(type)
+ let cli_opts = r |> unwrap_or_default
+ plan.jit_all_functions = prog.policies.jit_jit_all_functions
+ plan.debug_info = cli_opts.debug_info |> unwrap_or(prog.policies.jit_debug_info)
+ plan.dump_ir = cli_opts.dump_ir |> unwrap_or(false)
+ if (announce && prog.policies.jit_dll_mode && !das_is_dll_build()) {
+ // a legitimate configuration (static daslang), but the cost is worth naming: no DLL
+ // cache exists, so every -jit run pays full in-memory codegen
+ to_log(LOG_WARNING, "LLVM JIT: dll mode requested but this daslang build is static - no DLL cache, " +
+ "every run pays full in-memory codegen\n")
+ }
+ plan.gen_exe = prog.policies.jit_exe_mode
+ plan.exe_main = "main"
+ // --jit-compile-only: build + verify + optimize (+dump), write/load/install nothing —
+ // the program runs interpreted. Also bypasses the DLL cache probe, so the
+ // generators always run (this is the cross-emission smoke rail).
+ plan.compile_only = cli_opts.compile_only |> unwrap_or(false)
+ // Offline AOT: emit the whole used function set to a .o for static linking, with a
+ // load ctor registering them into the AOT library. Mutually exclusive with dll
+ // mode (no in-memory engine work, no instrument).
+ plan.emit_aot_object = prog.policies.jit_emit_object
+ plan.use_dll = prog.policies.jit_dll_mode && das_is_dll_build() && !plan.gen_exe && !plan.compile_only && !plan.emit_aot_object
+ // --jit-split-modules (or `options jit_split_modules`): -1 = per-das-module partitions, auto
+ // threads (default - the obj cache serves every program sharing a module); 0 = the monolith
+ // with full cross-module inlining, which -exe always emits (split is DLL-path only)
+ let split_option = prog._options |> find_arg("jit_split_modules")
+ plan.split_requested = (cli_opts.split_modules |> is_some()) || !(split_option is nothing)
+ plan.split_modules = cli_opts.split_modules |> unwrap_or(split_option ?as tInt ?? -1)
+ plan.jit_split = plan.split_modules != 0 && plan.use_dll
+ if (announce && plan.split_requested && plan.split_modules != 0 && !plan.use_dll) {
+ to_log(LOG_WARNING, "LLVM JIT: --jit-split-modules applies to the JIT DLL path only - ignored for this artifact mode\n")
+ }
+ // --jit-obj-cache (default ON under split): link cached partition objects for the unchanged
+ // module prefix. CLI wins over the script option. A pinned jit_output_path disables it:
+ // no GC runs there, so key-named objects would accumulate without bound in a user-owned dir.
+ let obj_cache_default = (prog._options |> find_arg("jit_obj_cache")) ?as tBool ?? true
+ plan.obj_cache = plan.jit_split && (string(prog.policies.jit_output_path) |> empty()) && (cli_opts.obj_cache |> unwrap_or(obj_cache_default ? 1 : 0)) != 0
+ if (announce && (cli_opts.obj_cache |> is_some()) && !plan.jit_split) {
+ to_log(LOG_WARNING, "LLVM JIT: --jit-obj-cache applies under --jit-split-modules dll mode only - ignored for this run\n")
+ }
+ plan.opt_level = cli_opts.opt_level |> unwrap_or(prog.policies.jit_opt_level)
+ plan.size_level = cli_opts.size_level |> unwrap_or(prog.policies.jit_size_level)
+ // This policy must be set before compilation so stack-frame sizes include the prologue.
+ plan.emit_prologue = prog.policies.emit_prologue
+ // If you're sure `dynamic` functions is not reachable set this flag
+ // to `true` and fail in runtime on `jit_trap` if they turns
+ // out to be reachable.
+ plan.exe_strict = true
+ plan.config_out_path = string(prog.policies.jit_output_path)
+ plan.output_path = plan.config_out_path |> empty() ? ".jitted_scripts/{prog.thisNamespace}" : plan.config_out_path
+ plan.path_to_shared_lib = string(prog.policies.jit_path_to_shared_lib)
+ // CLI --jit-path-to-linker wins over the script-level policy.
+ plan.path_to_linker = cli_opts.path_to_linker |> unwrap_or(string(prog.policies.jit_path_to_linker))
+ // Cross-compile triple: --jit-target= CLI wins over the
+ // script-level `options jit_target = "..."`.
+ let opt_target = (prog._options |> find_arg("jit_target")) ?as tString ?? ""
+ plan.target_triple = cli_opts.target |> unwrap_or(opt_target)
+ // the compile-time target folds (get_target_triple, get_target_architecture_name) read the
+ // command line, never this option: a script-level triple the command line does not carry
+ // would emit for one target with every tier gate folded for the host
+ if (!empty(plan.target_triple) && get_target_triple() != plan.target_triple) {
+ panic("LLVM JIT: the target triple `{plan.target_triple}` is not on the command line - pass --jit-target={plan.target_triple} after the `--` separator (the compile-time target folds read argv, so `options jit_target` alone folds the host's tiers into a cross artifact)")
+ }
+ plan.gen_wasm = plan.gen_exe && (plan.target_triple |> starts_with("wasm"))
+ // Optional CLI override for the wasm runtime archive location. Falls
+ // back to `options jit_runtime_lib = "..."` then write_wasm's
+ // find_runtime_lib() auto-locate under web/output/lib/.
+ plan.runtime_lib_override = cli_opts.runtime_lib |> unwrap_or((prog._options |> find_arg("jit_runtime_lib")) ?as tString ?? "")
+ // --jit-emit-object: emit the wasm object and skip the emcc link (the caller
+ // links it, e.g. daspkg release wasm). Sidesteps the link entirely, including
+ // the Windows popen-quoting failure path in link_wasm.
+ plan.emit_object = cli_opts.emit_object |> unwrap_or(false)
+ // --jit-threads: emit a -pthread-compatible wasm object (+atomics,+bulk-memory).
+ plan.wasm_threads = cli_opts.threads |> unwrap_or(false)
+ // --jit-check-abi: emit the handled-type host-vs-target layout safe-check into
+ // initialize_modules().
+ plan.check_abi = cli_opts.check_abi |> unwrap_or(false)
+ // Extra args appended verbatim to the host linker cmd. CLI > script option.
+ plan.linker_string = cli_opts.linker_string |> unwrap_or((prog._options |> find_arg("jit_linker_string")) ?as tString ?? "")
+ plan.register_all_modules = cli_opts.register_all_modules |> unwrap_or(false)
+
+ // Set global options
+ LLVM_JIT_ALLOW_UNALIGNED_VECTOR_READ_OUT_OF_BOUNDS = prog._options |> find_arg("jit_vec3_ldu") ?as tBool ?? LLVM_JIT_ALLOW_UNALIGNED_VECTOR_READ_OUT_OF_BOUNDS_DEFAULT
+ // `options fast_math` (or policies.fast_math) is the one knob; `_jit_fast_math` kept as a legacy alias
+ // a runtime built -ffast-math already relaxes the interpreter's own ops, so the JIT inherits
+ // that unless the program says otherwise; `options fast_math` still wins either way
+ g_jit_fast_math = (prog._options |> find_arg("fast_math") ?as tBool ?? (prog.policies.fast_math || HOST_FAST_MATH)) || (prog._options |> find_arg("_jit_fast_math") ?as tBool ?? false)
+
+ // nolint:STYLE014
+ // Standalone exes target a GENERIC CPU (write_exe use_host_cpu=false default), so
+ // host cpuid features must not leak into their emission (host_features=false) —
+ // EXCEPT a native exe carrying [llvm_code] kernels: a local-use artifact by
+ // definition, it targets the current box in BOTH the flags and the machine (the ONE
+ // truth, decided here — flags and machine must never diverge: host-F16C flags on a
+ // generic machine lower fp16 through libcalls msvcrt never links, and generic flags
+ // on a host machine decline every generated kernel to its reference body).
+ // A baseline class (DAS_JIT_BASELINE) replaces the box on every rail - exe, DLL and AOT object
+ // alike: the machine, the gates and the tune ladder all read the class, so the artifact runs
+ // on every box of that class. It never crosses architectures; that is --jit-target's job.
+ plan.baseline = jit_baseline_class()
+ if (!empty(plan.baseline)) {
+ let target_arch = (empty(plan.target_triple) ? get_architecture_name()
+ : ((starts_with(plan.target_triple, "aarch64") || starts_with(plan.target_triple, "arm64")) ? "arm64"
+ : ((starts_with(plan.target_triple, "x86_64") || starts_with(plan.target_triple, "amd64")) ? "x86_64" : plan.target_triple)))
+ let berr = jit_baseline_error(target_arch)
+ if (!empty(berr)) {
+ panic("LLVM JIT: {berr}")
+ }
+ if (announce) {
+ let row = jit_cpu_class_row(plan.baseline)
+ to_log(LOG_INFO, "LLVM JIT: DAS_JIT_BASELINE={plan.baseline} - targeting the {plan.baseline} class (cpu {row.cpu}, features {row.features}) instead of this box\n")
+ }
+ }
+ plan.exe_host_cpu = plan.gen_exe && empty(plan.target_triple) && empty(plan.baseline) && has_generated_kernel(prog, plan.jit_all_functions)
+ if (announce && plan.exe_host_cpu) {
+ to_log(LOG_INFO, "LLVM EXE: targeting the current box (CPU + features) - the build carries [llvm_code] kernels\n")
+ }
+ // AOT objects target the host CPU: the .o is statically linked into a binary that runs on this
+ // box (same contract as the JIT), so bake host features — this force-appends +dotprod on aarch64,
+ // without which the dasLLAMA Q8 SDOT kernels have no instruction to select and codegen aborts.
+ plan.use_host_cpu = empty(plan.baseline) && (plan.emit_aot_object ? true : (!plan.gen_exe || plan.exe_host_cpu))
+ plan.log_jit_time = prog._options |> find_arg("log_compile_time") ?as tBool ?? prog.policies.log_compile_time
+
+ prog |> for_each_module() $(mod) {
+ mod |> for_each_function("") $(fun) {
+ // In object mode emit the whole used, non-noAot set (all modules +
+ // per-program generic instantiations), mirroring C++ AOT whole-program
+ // coverage; noAot functions interpret as Program::linkCppAot skips them.
+ if (is_used(prog, fun) && (!plan.emit_aot_object || !fun.flags.noAot)
+ && get_function_by_mangled_name_hash(hash(get_mangled_name(fun)), *ctx) != null
+ && jit_selects(fun, fun.moreFlags.requestJit, plan.aot_host, plan.jit_all_functions, ctx)) {
+ if (!fun.moreFlags.requestNoJit) {
+ plan.candidates |> emplace(fun)
+ } else {
+ plan.disabled |> emplace(fun)
+ }
+ }
+ }
+ }
+ // Content-address the DLL: distinct (AST + codegen version + opt flags) → distinct files, same
+ // inputs → same filename → cache hit. Only in dll-mode with the default output path; a pinned
+ // jit_output_path or an exe keeps the user's path.
+ if (plan.use_dll && plan.config_out_path |> empty()) {
+ plan.dll_hash_basename = jit_dll_basename(plan.candidates, plan.opt_level, plan.size_level, plan.emit_prologue, plan.debug_info, plan.target_triple, plan.jit_split, plan.use_host_cpu)
+ plan.output_path = ".jitted_scripts/{prog.thisNamespace}/{plan.dll_hash_basename}"
+ }
+ return <- plan
+}
diff --git a/modules/dasLLVM/daslib/llvm_jit_run.das b/modules/dasLLVM/daslib/llvm_jit_run.das
index 726788f90b..34fb269791 100644
--- a/modules/dasLLVM/daslib/llvm_jit_run.das
+++ b/modules/dasLLVM/daslib/llvm_jit_run.das
@@ -6,15 +6,14 @@ options no_global_variables = false
require llvm/daslib/llvm_boost
require llvm/daslib/llvm_dll_utils
+require llvm/daslib/llvm_jit_plan public
require llvm/daslib/llvm_exe
require llvm/daslib/llvm_aot
require llvm/daslib/llvm_jit
require llvm/daslib/llvm_jit_common
require llvm/daslib/llvm_jit_di
require llvm/daslib/llvm_jit_intrin
-require llvm/daslib/llvm_jit_cli
require llvm/daslib/llvm_env
-require llvm/daslib/llvm_cpu_class
require daslib/ast_boost
require daslib/strings_boost
@@ -34,23 +33,9 @@ module llvm_jit_run shared private
var LINK_WHOLE_LIB = false // when true, standalone exe links against the whole library; when false, only against runtime
-// nolint:STYLE014
-// Bump this constant whenever LLVM codegen / linking logic changes in a way that
-// invalidates cached DLLs (e.g. edits to llvm_jit.das, llvm_macro.das, llvm_jit_common.das,
-// runtime helper ABI, default target triple). Cache filenames fold this in, so a bump
-// makes every previously written DLL miss the cache on the next run and get GC'd.
-let LLVM_JIT_CODEGEN_VERSION : uint64 = 0x7cul // under --jit-debug a das global is described too, at the ctx-derived pointer each function computes for it; every das local, argument and block variable carries a DILocalVariable with a real DIType, a das block opens a DILexicalBlock, a subprogram is named as the das source names it and carries no linkage name, one compile unit covers the whole module, and the prologue and a loop latch carry no stale line
-
// Read by tests-cpp/small/test_jit_emitter_pin.cpp: FNV-1a64 of the emitter sources
// (normalized to LF; file list in the test)
-let LLVM_JIT_EMITTER_HASH : uint64 = 0x65480ff16be6cbadul
-
-let JIT_FNV_PRIME : uint64 = 1099511628211ul
-
-// `options fast_math` stamps every FP op with VALUE-SAFE flags (reassoc|nsz|contract) before opt —
-// NON-bit-exact, default off, +8-10% e2e. nnan|ninf|arcp|afn are deliberately NOT stamped: engine
-// kernels make real infs and arcp+ninf's rcp-estimate div computes inf*0=NaN (tests/jit/fast_math_specials.das).
-var g_jit_fast_math = false
+let LLVM_JIT_EMITTER_HASH : uint64 = 0x79e488af0ec6e623ul
def private apply_fast_math_to_module(m : LLVMOpaqueModule?) {
var fn = LLVMGetFirstFunction(m)
@@ -71,239 +56,6 @@ def private apply_fast_math_to_module(m : LLVMOpaqueModule?) {
}
}
-// nolint:STYLE014
-// Loop tune hints (vectorize / vectorize_width / unroll / unroll_count / interleave_count)
-// live on ExprFor / ExprWhile annotations and reach ONLY the JIT — attach_loop_metadata
-// turns them into LLVM !llvm.loop metadata. They never enter the interpreter SimNode tree
-// that get_function_aot_hash walks, so the semantic hash is blind to them: a re-tuned
-// kernel ([tuned] restamping a different perm from a changed box_profile.json) has an
-// IDENTICAL aot hash and would silently reuse a stale DLL. Fold them into the JIT-only
-// cache key here, alongside LLVM_JIT_CODEGEN_VERSION / g_jit_fast_math.
-// Fold the ONE value field the arg's type says is live. bValue/iValue/fValue share a C++
-// union — reading a non-live member is UB (a bool load of vectorize_width=8 aborts ubsan
-// and lets clang -O2 miscompile: chunk workers died with SIGSEGV on linux Release CI).
-// Type coverage stays total: any basicType re-keys the DLL via its own live field.
-def private fold_annotation_args(args : AnnotationArgumentList; seed : uint64) : uint64 {
- var h = seed
- for (a in args) {
- h = (h ^ hash(a.name)) * JIT_FNV_PRIME
- h = (h ^ uint64(int(a.basicType))) * JIT_FNV_PRIME
- if (a.basicType == Type.tBool) {
- h = (h ^ (a.bValue ? 1ul : 0ul)) * JIT_FNV_PRIME
- } elif (a.basicType == Type.tInt) {
- h = (h ^ uint64(a.iValue)) * JIT_FNV_PRIME
- } elif (a.basicType == Type.tFloat) {
- h = (h ^ hash(a.fValue)) * JIT_FNV_PRIME
- } elif (a.basicType == Type.tString) {
- h = (h ^ hash(a.sValue)) * JIT_FNV_PRIME
- }
- }
- return h
-}
-
-class private LoopHintHashVisitor : AstVisitor {
- h : uint64
- def override preVisitExprFor(expr : ExprFor?) : void {
- h = fold_annotation_args(expr.annotations, h)
- }
- def override preVisitExprWhile(expr : ExprWhile?) : void {
- h = fold_annotation_args(expr.annotations, h)
- }
-}
-
-// Complete under jit_all_functions (the default): every used function is a top-level entry
-// in `funcs`, so folding each function's own body covers the whole compiled program.
-def private fold_loop_hints(funcs : array; seed : uint64) : uint64 {
- var vis = new LoopHintHashVisitor()
- vis.h = seed
- make_visitor(*vis) $(adapter) {
- for (fn in funcs) {
- visit(fn.body, adapter)
- }
- }
- let r = vis.h
- unsafe { delete vis }
- return r
-}
-
-// [llvm_code] annotation args are generator params: they change emitted IR without touching the
-// reference body, so the aot hash is blind — fold them like loop hints. A generator CODE change is
-// invisible too: bump LLVM_JIT_CODEGEN_VERSION when a generator's emission changes for fixed args.
-def private fold_llvm_code_hints(funcs : array; seed : uint64) : uint64 {
- var h = seed
- for (fn in funcs) {
- for (ann in fn.annotations) {
- if (ann.annotation.name == "llvm_code") {
- h = (h ^ hash("llvm_code")) * JIT_FNV_PRIME
- h = fold_annotation_args(ann.arguments, h)
- }
- }
- }
- return h
-}
-
-// [hint(...)] args become LLVM function attributes (alwaysinline/noalias=/optnone/...) — backend-only,
-// no SimNode, so the aot hash is blind to them exactly like loop hints; fold them into both keys
-def private fold_function_hints(funcs : array; seed : uint64) : uint64 {
- var h = seed
- for (fn in funcs) {
- for (ann in fn.annotations) {
- if (ann.annotation.name == "hint") {
- h = (h ^ hash("hint")) * JIT_FNV_PRIME
- h = fold_annotation_args(ann.arguments, h)
- }
- }
- }
- return h
-}
-
-// True when the program jits any [llvm_code] kernel — those emit host-specific IR (e.g. AVX2) a
-// generic x86-64 exe can't legalize, so a -exe carrying them must target the current box. Scanned
-// BEFORE target flags: the generators' decline predicates consult g_target_*, so order matters.
-def private has_generated_kernel(prog : Program?; jit_all_functions : bool) : bool {
- var found = false
- prog |> for_each_module() $(mod) {
- mod |> for_each_function("") $(fun) {
- if (fun.flags.used && (jit_all_functions || fun.moreFlags.requestJit) && !fun.moreFlags.requestNoJit) {
- for (ann in fun.annotations) {
- if (ann.annotation.name == "llvm_code") {
- found = true
- }
- }
- }
- }
- }
- return found
-}
-
-// The environment/config salt shared by the whole-DLL key AND the per-partition obj cache key.
-// ONE helper on purpose: a salt component folded into the DLL key but not the partition key
-// would let a config change reuse stale cached partition objects.
-def jit_env_salt(opt_level : int; size_level : int; emit_prologue : bool; debug_info : bool; // nolint:STYLE037,STYLE038 - one flat fold per emission input; a split would hide which inputs key the cache
- target_triple : string; split : bool; use_host_cpu : bool) : uint64 {
- var h = LLVM_JIT_CODEGEN_VERSION
- // host/binary identity: object format follows the triple (COFF vs ELF on one checkout), and the emitting binary bakes its Context offsets into every global access - neither is in any AST hash
- if (empty(target_triple)) {
- let host_triple = LLVMGetDefaultTargetTriple()
- h = (h ^ hash(host_triple)) * JIT_FNV_PRIME
- LLVMDisposeMessage(host_triple)
- }
- h = (h ^ uint64(CONTEXT_OFFSET_OF_GLOBALS)) * JIT_FNV_PRIME
- h = (h ^ uint64(CONTEXT_OFFSET_OF_SHARED)) * JIT_FNV_PRIME
- h = (h ^ uint64(CONTEXT_OFFSET_OF_STOP_FLAGS)) * JIT_FNV_PRIME
- h = (h ^ uint64(CONTEXT_OFFSET_OF_EVAL_TOP)) * JIT_FNV_PRIME
- // full host-CPU identity when the TargetMachine bakes it - the cpuid bits below cover only the matrix-tier features; a wider box's object is an illegal instruction on a narrower one
- if (use_host_cpu) {
- let host_cpu = LLVMGetHostCPUName()
- let host_features = LLVMGetHostCPUFeatures()
- h = (h ^ hash(host_cpu)) * JIT_FNV_PRIME
- h = (h ^ hash(host_features)) * JIT_FNV_PRIME
- LLVMDisposeMessage(host_cpu)
- LLVMDisposeMessage(host_features)
- }
- // DAS_JIT_PROBE_LTO artifacts (bitcode + lld-LTO link) must never serve a normal run
- if (g_env_jit.jit_probe_lto) {
- h = (h ^ hash("lto_probe")) * JIT_FNV_PRIME
- }
- // options jit_vec3_ldu selects the vector-load emission shape
- h = (h ^ (LLVM_JIT_ALLOW_UNALIGNED_VECTOR_READ_OUT_OF_BOUNDS ? 1ul : 0ul)) * JIT_FNV_PRIME
- h = (h ^ uint64(opt_level)) * JIT_FNV_PRIME
- h = (h ^ uint64(size_level)) * JIT_FNV_PRIME
- h = (h ^ (emit_prologue ? 1ul : 0ul)) * JIT_FNV_PRIME
- h = (h ^ (debug_info ? 1ul : 0ul)) * JIT_FNV_PRIME
- h = (h ^ (g_jit_fast_math ? 1ul : 0ul)) * JIT_FNV_PRIME // fast-math changes codegen → distinct cache
- // split-built and monolith DLLs differ in inlining — never share a cache entry (bool only: partition count is data-driven)
- h = (h ^ (split ? 1ul : 0ul)) * JIT_FNV_PRIME
- // EH vs longjmp hosts emit different unwind tables — must not share a cache entry (panic would crash)
- h = (h ^ (das_is_exceptions_enabled() ? 1ul : 0ul)) * JIT_FNV_PRIME
- // AVX2 gates the x64 intrinsic table — an AVX2-jitted DLL must not cache-hit an AVX2-less host (illegal instr)
- h = (h ^ (cpu_supports("avx2") ? 1ul : 0ul)) * JIT_FNV_PRIME
- // Matrix-tier cpuid bits (vnni256/avx512bw/avx512vnni) + DAS_JIT_X64_FORCE_FEATURES key emission the same way
- var isa = 0ul
- if (cpu_supports("avxvnni")) {
- isa |= 1ul
- }
- if (cpu_supports("avx512f")) {
- isa |= 2ul
- }
- if (cpu_supports("avx512bw")) {
- isa |= 4ul
- }
- if (cpu_supports("avx512vl")) {
- isa |= 8ul
- }
- if (cpu_supports("avx512vnni")) {
- isa |= 16ul
- }
- if (cpu_supports("avx512vbmi")) { // the grid formats' symbol lattice emits on it (grid = "vbmi")
- h = (h ^ 0x1Bul) * JIT_FNV_PRIME
- }
- if (cpu_supports("amx-tile") && cpu_supports("amx-int8")) { // tune_cpu_class picks x86-amx, which selects grid decode forms
- h = (h ^ 0x1Cul) * JIT_FNV_PRIME
- }
- if (cpu_supports("avxvnniint8")) {
- isa |= 32ul
- }
- h = (h ^ isa) * JIT_FNV_PRIME
- let forced = g_env_jit.jit_x64_force_features
- if (!empty(forced)) {
- h = (h ^ hash(forced)) * JIT_FNV_PRIME
- }
- // aarch64 twin (DAS_JIT_ARM64_FORCE_FEATURES, e.g. "i8mm"): a forced artifact must not cache-hit a normal run
- let forced_arm = g_env_jit.jit_arm64_force_features
- if (!empty(forced_arm)) {
- h = (h ^ hash(forced_arm)) * JIT_FNV_PRIME
- }
- // aarch64 i8mm is an emission input (the +i8mm append): a warm pre-i8mm cache must not serve an i8mm host
- h = (h ^ (cpu_supports("i8mm") ? 64ul : 0ul)) * JIT_FNV_PRIME
- // a baseline class (DAS_JIT_BASELINE) picks the machine, the gates and the tune ladder - its artifact never serves a host run
- let baseline = jit_baseline_class()
- if (!empty(baseline)) {
- h = (h ^ hash(baseline)) * JIT_FNV_PRIME
- }
- // Cross-compile artifacts key on their triple — a --jit-target DLL must not cache-hit a host run (foreign-arch dlopen)
- if (!empty(target_triple)) {
- h = (h ^ hash(target_triple)) * JIT_FNV_PRIME
- }
- return h
-}
-
-def jit_dll_basename(funcs : array; opt_level : int; size_level : int;
- emit_prologue : bool; debug_info : bool; target_triple : string; split : bool;
- use_host_cpu : bool) : string {
- var h = LLVM_JIT_CODEGEN_VERSION
- for (fn in funcs) {
- h = (h ^ get_function_aot_hash(fn)) * JIT_FNV_PRIME
- }
- h = fold_loop_hints(funcs, h) // JIT-only loop tune hints (see LoopHintHashVisitor)
- h = fold_llvm_code_hints(funcs, h) // [llvm_code] generator params (annotation args)
- h = fold_function_hints(funcs, h) // [hint(...)] function attributes (backend-only, aot-hash-blind)
- h = (h ^ jit_env_salt(opt_level, size_level, emit_prologue, debug_info, target_triple, split, use_host_cpu)) * JIT_FNV_PRIME
- return "{h}"
-}
-
-def jit_gc_stale_dlls(subdir : string; keep_basename : string; keep_names : table) {
- // keep mask: the current DLL-hash prefix (dll/lib/exp/pdb/rsp + monolith .o) OR an exact keep_names entry (split obj cache's live partition objects - named by partition key, not dll hash)
- let keep_prefix = "{keep_basename}."
- dir(subdir) $(fname) {
- if (fname == "." || fname == ".." || fname |> starts_with(keep_prefix) || keep_names |> key_exists(fname)) return
- let full_path = "{subdir}/{fname}"
- var err : string
- let ok = remove(full_path, err)
- if (ok && empty(err)) {
- to_log(LOG_INFO, "LLVM JIT: GC stale artifact {full_path}\n")
- } else {
- to_log(LOG_WARNING, "LLVM JIT: failed to GC stale artifact {full_path}: {err}\n")
- }
- }
-}
-
-struct JitState {
- dll : void?
- ee : void?
- ctx : void?
-}
-
// reopen the freshly written DLL for the install phase + GC stale artifacts - the ONE tail both
// the monolith and split arms share (the keep-mask decides which cached artifacts survive)
[macro_function]
@@ -394,32 +146,6 @@ def reset_jit_globals_after_failure() {
reset_codegen_accumulators()
}
-// always-on summary (a silent multi-second jit is indistinguishable from a hang) + the
-// log_compile_time per-phase breakdown; both artifact paths report through here
-def private jit_log_report(funcs : array; recompiled : bool; opt_level : int; tier_known : bool;
- split : bool; split_parts : int; debug_info : bool; log_time : bool; totalTime : int64;
- t_hash, t_init, t_declare, t_probe, t_irgen, t_opt, t_emit, t_install, t_finalize : int) {
- var tuneStamped = 0
- for (fn in funcs) {
- for (ann in fn.annotations) {
- if (ann.annotation.name == "llvm_code" && find_arg(ann.arguments, "tune_suffix") is tString) {
- tuneStamped++
- break
- }
- }
- }
- let tuneTail = tuneStamped > 0 ? " incl {tuneStamped} tune-stamped" : ""
- let tierTail = tier_known ? ", O{opt_level}" : ""
- // always announce split - a split-built DLL's profile is not comparable to a monolith build
- let splitTail = !split ? "" : (split_parts > 0 ? ", split {split_parts} modules" : ", split")
- let debugTail = debug_info ? ", --jit-debug" : ""
- let totalDt = double(get_time_usec(totalTime)) / 1000000.0lf
- to_log(LOG_INFO, "LLVM JIT: {length(funcs)} functions{tuneTail} in {totalDt} sec ({recompiled ? "codegen" : "cache"}{tierTail}{splitTail}{debugTail})\n")
- if (log_time) {
- to_log(LOG_INFO, "LLVM JIT time: hash {jit_sec(t_hash)} init {jit_sec(t_init)} declare {jit_sec(t_declare)} probe {jit_sec(t_probe)} irgen {jit_sec(t_irgen)} optimize {jit_sec(t_opt)} emit+link {jit_sec(t_emit)} install {jit_sec(t_install)} finalize {jit_sec(t_finalize)}\n")
- }
-}
-
// "Debug Info Version" gates all !dbg metadata (absent -> silently stripped); "CodeView" flips
// COFF debug emission from DWARF to CodeView so lld-link /DEBUG can build PDB line tables
// (windows targets only). Per-module flags: split mode stamps every partition module.
@@ -444,7 +170,7 @@ def private irgen_functions(ctx : Context?; var uids : UidNodes?; attrs : Attrib
prog : Program?) : bool {
var has_externals = false
for (fun in part) {
- has_externals = generate_llvm(ctx, g_prim_t, uids, attrs, fun, jit_flags, jit_mode) || has_externals
+ has_externals = generate_llvm(prog, ctx, g_prim_t, uids, attrs, fun, jit_flags, jit_mode) || has_externals
let fnmna = uids.get_dll_fn_name(fun)
var fn_impl = LLVMGetNamedFunction(g_mod, fnmna.impl())
var fn = LLVMGetNamedFunction(g_mod, fnmna.publ())
@@ -606,7 +332,7 @@ def private run_split_codegen(prog : Program?; ctx : Context?; funcs : arrayadd_llvm_functions(fun)
@@ -706,217 +432,99 @@ def private run_split_codegen(prog : Program?; ctx : Context?; funcs : array
- var disabled : array
- var visitorDisabled = 0
- var disableJitVisitor = new DisableJitVisitor()
-
- let r <- parse_args(type)
- let cli_opts = r |> unwrap_or_default
- assume jit_all_functions = prog.policies.jit_jit_all_functions
- let debug_info = cli_opts.debug_info |> unwrap_or(prog.policies.jit_debug_info)
- let dump_ir = cli_opts.dump_ir |> unwrap_or(false)
- if (prog.policies.jit_dll_mode && !das_is_dll_build()) {
- // a legitimate configuration (static daslang), but the cost is worth naming: no DLL
- // cache exists, so every -jit run pays full in-memory codegen
- to_log(LOG_WARNING, "LLVM JIT: dll mode requested but this daslang build is static - no DLL cache, " +
- "every run pays full in-memory codegen\n")
- }
- let gen_exe = prog.policies.jit_exe_mode
+ let totalTime = ref_time_ticks()
+ var phase_tm = ref_time_ticks()
+ var plan <- make_jit_plan(prog, ctx, false)
+ assume aot_host = plan.aot_host
+ assume jit_all_functions = plan.jit_all_functions
+ assume debug_info = plan.debug_info
+ assume dump_ir = plan.dump_ir
+ assume gen_exe = plan.gen_exe
+ assume exe_main = plan.exe_main
+ assume compile_only = plan.compile_only
+ assume emit_aot_object = plan.emit_aot_object
+ assume use_dll = plan.use_dll
+ assume split_modules = plan.split_modules
+ assume jit_split = plan.jit_split
+ assume obj_cache = plan.obj_cache
+ assume opt_level = plan.opt_level
+ assume size_level = plan.size_level
+ assume emit_prologue = plan.emit_prologue
+ assume exe_strict = plan.exe_strict
+ assume config_out_path = plan.config_out_path
+ assume output_path = plan.output_path
+ assume path_to_shared_lib = plan.path_to_shared_lib
+ assume path_to_linker = plan.path_to_linker
+ assume target_triple = plan.target_triple
+ assume gen_wasm = plan.gen_wasm
+ assume runtime_lib_override = plan.runtime_lib_override
+ assume emit_object = plan.emit_object
+ assume linker_string = plan.linker_string
+ assume exe_host_cpu = plan.exe_host_cpu
+ assume use_host_cpu = plan.use_host_cpu
+ assume log_jit_time = plan.log_jit_time
+ assume dll_hash_basename = plan.dll_hash_basename
+ assume disabled = plan.disabled
// Drives the $::is_standalone_exe JIT intrinsic in llvm_jit_intrin so daslang
// code can constant-fold to true when this codegen run produces a standalone exe.
llvm_jit_intrin::g_jit_exe_mode = gen_exe
defer() { llvm_jit_intrin::g_jit_exe_mode = false; }
- let exe_main = "main" // prog.policies.jit_entry_point |> empty() ? "main" : prog.policies.jit_entry_point
- // --jit-compile-only: build + verify + optimize (+dump), write/load/install nothing —
- // the program runs interpreted. Also bypasses the DLL cache probe below, so the
- // generators always run (this is the cross-emission smoke rail).
- let compile_only = cli_opts.compile_only |> unwrap_or(false)
- // Offline AOT: emit the whole used function set to a .o for static linking, with a
- // load ctor registering them into the AOT library. Mutually exclusive with dll
- // mode (no in-memory engine work, no instrument).
- let emit_aot_object = prog.policies.jit_emit_object
// Emit builtin-accessor globals internal + null in object mode (re-resolved by
// the emit_aot_object_globinit ctor); reset after this run.
g_emit_aot_object_globals = emit_aot_object
- if (emit_aot_object) {
- }
defer() { g_emit_aot_object_globals = false; }
- let use_dll = prog.policies.jit_dll_mode && das_is_dll_build() && !gen_exe && !compile_only && !emit_aot_object
- // --jit-split-modules (or `options jit_split_modules`): -1 = per-das-module partitions, auto
- // threads (default - the obj cache serves every program sharing a module); 0 = the monolith
- // with full cross-module inlining, which -exe always emits (split is DLL-path only)
- let split_option = prog._options |> find_arg("jit_split_modules")
- let split_requested = (cli_opts.split_modules |> is_some()) || !(split_option is nothing)
- let split_modules = cli_opts.split_modules |> unwrap_or(split_option ?as tInt ?? -1)
- let jit_split = split_modules != 0 && use_dll
- if (split_requested && split_modules != 0 && !use_dll) {
- to_log(LOG_WARNING, "LLVM JIT: --jit-split-modules applies to the JIT DLL path only - ignored for this artifact mode\n")
- }
g_jit_split_active = jit_split
defer() { g_jit_split_active = false; }
- // --jit-obj-cache (default ON under split): link cached partition objects for the unchanged
- // module prefix. CLI wins over the script option. A pinned jit_output_path disables it:
- // no GC runs there, so key-named objects would accumulate without bound in a user-owned dir.
- let obj_cache_default = (prog._options |> find_arg("jit_obj_cache")) ?as tBool ?? true
- let obj_cache = jit_split && (string(prog.policies.jit_output_path) |> empty()) && (cli_opts.obj_cache |> unwrap_or(obj_cache_default ? 1 : 0)) != 0
- if ((cli_opts.obj_cache |> is_some()) && !jit_split) {
- to_log(LOG_WARNING, "LLVM JIT: --jit-obj-cache applies under --jit-split-modules dll mode only - ignored for this run\n")
- }
- let opt_level = cli_opts.opt_level |> unwrap_or(prog.policies.jit_opt_level)
- let size_level = cli_opts.size_level |> unwrap_or(prog.policies.jit_size_level)
- // This policy must be set before compilation so stack-frame sizes include the prologue.
- let emit_prologue = prog.policies.emit_prologue
- // If you're sure `dynamic` functions is not reachable set this flag
- // to `true` and fail in runtime on `jit_trap` if they turns
- // out to be reachable.
- let exe_strict = true
- assume config_out_path = string(prog.policies.jit_output_path)
- var output_path = config_out_path |> empty() ? ".jitted_scripts/{prog.thisNamespace}" : config_out_path
- assume path_to_shared_lib = string(prog.policies.jit_path_to_shared_lib)
- // CLI --jit-path-to-linker wins over the script-level policy.
- let path_to_linker = cli_opts.path_to_linker |> unwrap_or(string(prog.policies.jit_path_to_linker))
- // Cross-compile triple: --jit-target= CLI wins over the
- // script-level `options jit_target = "..."`.
- let opt_target = (prog._options |> find_arg("jit_target")) ?as tString ?? ""
- let target_triple = cli_opts.target |> unwrap_or(opt_target)
- // the compile-time target folds (get_target_triple, get_target_architecture_name) read the
- // command line, never this option: a script-level triple the command line does not carry
- // would emit for one target with every tier gate folded for the host
- if (!empty(target_triple) && get_target_triple() != target_triple) {
- panic("LLVM JIT: the target triple `{target_triple}` is not on the command line - pass --jit-target={target_triple} after the `--` separator (the compile-time target folds read argv, so `options jit_target` alone folds the host's tiers into a cross artifact)")
- }
- let gen_wasm = gen_exe && (target_triple |> starts_with("wasm"))
- // Optional CLI override for the wasm runtime archive location. Falls
- // back to `options jit_runtime_lib = "..."` then write_wasm's
- // find_runtime_lib() auto-locate under web/output/lib/.
- let runtime_lib_override = cli_opts.runtime_lib |> unwrap_or((prog._options |> find_arg("jit_runtime_lib")) ?as tString ?? "")
- // --jit-emit-object: emit the wasm object and skip the emcc link (the caller
- // links it, e.g. daspkg release wasm). Sidesteps the link entirely, including
- // the Windows popen-quoting failure path in link_wasm.
- let emit_object = cli_opts.emit_object |> unwrap_or(false)
- // --jit-threads: emit a -pthread-compatible wasm object (+atomics,+bulk-memory).
- // Set the codegen global before any IR is built so both the data-layout machine
+ // --jit-threads: set the codegen global before any IR is built so both the data-layout machine
// and write_wasm's emission machine pick up the threaded feature set.
- g_target_wasm_threads = cli_opts.threads |> unwrap_or(false)
- // --jit-check-abi: emit the handled-type host-vs-target layout safe-check into
- // initialize_modules(). Set before codegen so llvm_exe's sweep sees the flag.
- g_jit_check_handled_abi = cli_opts.check_abi |> unwrap_or(false)
- // Extra args appended verbatim to the host linker cmd. CLI > script option.
- let linker_string = cli_opts.linker_string |> unwrap_or((prog._options |> find_arg("jit_linker_string")) ?as tString ?? "")
-
- // Set global options
- LLVM_JIT_ALLOW_UNALIGNED_VECTOR_READ_OUT_OF_BOUNDS = prog._options |> find_arg("jit_vec3_ldu") ?as tBool ?? LLVM_JIT_ALLOW_UNALIGNED_VECTOR_READ_OUT_OF_BOUNDS_DEFAULT
- // `options fast_math` (or policies.fast_math) is the one knob; `_jit_fast_math` kept as a legacy alias
- // a runtime built -ffast-math already relaxes the interpreter's own ops, so the JIT inherits
- // that unless the program says otherwise; `options fast_math` still wins either way
- g_jit_fast_math = (prog._options |> find_arg("fast_math") ?as tBool ?? (prog.policies.fast_math || HOST_FAST_MATH)) || (prog._options |> find_arg("_jit_fast_math") ?as tBool ?? false)
-
- // nolint:STYLE014
+ g_target_wasm_threads = plan.wasm_threads
+ // --jit-check-abi: set before codegen so llvm_exe's sweep sees the flag.
+ g_jit_check_handled_abi = plan.check_abi
// the disable pass consults target-gated intrinsic tables (has_intrinsic) — the
// g_target_* truth must be set BEFORE it runs, not first at init_jit time below.
- // Standalone exes target a GENERIC CPU (write_exe use_host_cpu=false default), so
- // host cpuid features must not leak into their emission (host_features=false) —
- // EXCEPT a native exe carrying [llvm_code] kernels: a local-use artifact by
- // definition, it targets the current box in BOTH the flags and the machine (the ONE
- // truth, decided here — flags and machine must never diverge: host-F16C flags on a
- // generic machine lower fp16 through libcalls msvcrt never links, and generic flags
- // on a host machine decline every generated kernel to its reference body).
- // A baseline class (DAS_JIT_BASELINE) replaces the box on every rail - exe, DLL and AOT object
- // alike: the machine, the gates and the tune ladder all read the class, so the artifact runs
- // on every box of that class. It never crosses architectures; that is --jit-target's job.
- let baseline = jit_baseline_class()
- if (!empty(baseline)) {
- let target_arch = (empty(target_triple) ? get_architecture_name()
- : ((starts_with(target_triple, "aarch64") || starts_with(target_triple, "arm64")) ? "arm64"
- : ((starts_with(target_triple, "x86_64") || starts_with(target_triple, "amd64")) ? "x86_64" : target_triple)))
- let berr = jit_baseline_error(target_arch)
- if (!empty(berr)) {
- panic("LLVM JIT: {berr}")
- }
- let row = jit_cpu_class_row(baseline)
- to_log(LOG_INFO, "LLVM JIT: DAS_JIT_BASELINE={baseline} - targeting the {baseline} class (cpu {row.cpu}, features {row.features}) instead of this box\n")
- }
- let exe_host_cpu = gen_exe && empty(target_triple) && empty(baseline) && has_generated_kernel(prog, jit_all_functions)
- if (exe_host_cpu) {
- to_log(LOG_INFO, "LLVM EXE: targeting the current box (CPU + features) - the build carries [llvm_code] kernels\n")
- }
- // AOT objects target the host CPU: the .o is statically linked into a binary that runs on this
- // box (same contract as the JIT), so bake host features — this force-appends +dotprod on aarch64,
- // without which the dasLLAMA Q8 SDOT kernels have no instruction to select and codegen aborts.
- let use_host_cpu = empty(baseline) && (emit_aot_object ? true : (!gen_exe || exe_host_cpu))
init_jit_target_flags(target_triple, use_host_cpu)
+ var funcs : array
+ var visitorDisabled = 0
+ var disableJitVisitor = new DisableJitVisitor()
make_visitor(*disableJitVisitor) $(disableJitVisitorAdapter) {
- prog |> for_each_module() $(mod) {
- mod |> for_each_function("") $(fun) {
- let rqj = fun.moreFlags.requestJit
- fun.moreFlags.requestJit = false
- // In object mode emit the whole used, non-noAot set (all modules +
- // per-program generic instantiations), mirroring C++ AOT whole-program
- // coverage; noAot functions interpret as Program::linkCppAot skips them.
- if (fun.flags.used && (!emit_aot_object || !fun.flags.noAot) && jit_selects(fun, rqj, aot_host, jit_all_functions, ctx)) {
- if (!fun.moreFlags.requestNoJit) {
- disableJitVisitor.disable = false
- // if DisableJitVisitor ever grows a preVisitFunction check again, it must
- // be dispatched explicitly here — `visit(fun)` never calls preVisitFunction
- fun |> visit(disableJitVisitorAdapter)
- if (!disableJitVisitor.disable) {
- fun.moreFlags.requestJit = true
- funcs |> emplace(fun)
- } else {
- visitorDisabled++
- to_log(LOG_INFO, "LLVM JIT: disabled {fun.name}\n")
- }
- } else {
- disabled |> emplace(fun)
- }
- }
+ for (fun in plan.candidates) {
+ disableJitVisitor.disable = false
+ // if DisableJitVisitor ever grows a preVisitFunction check again, it must
+ // be dispatched explicitly here — `visit(fun)` never calls preVisitFunction
+ fun |> visit(disableJitVisitorAdapter)
+ if (!disableJitVisitor.disable) {
+ funcs |> push(fun)
+ } else {
+ visitorDisabled++
+ to_log(LOG_INFO, "LLVM JIT: disabled {fun.name}\n")
}
}
}
unsafe {
delete disableJitVisitor
}
+ mark_jit_selection(prog, funcs)
// emit_aot_object always writes an object even with no functions to emit (a fully-no_aot
// module): an empty .o with just the fileinfo ctor, so the build always has the file it
// expects and those functions interpret at load (as Program::linkCppAot skips them).
if (!empty(funcs) || emit_aot_object) {
- let totalTime = ref_time_ticks()
- // the front end's log_compile_time option extends into the backend: per-phase wall below
- let log_jit_time = prog._options |> find_arg("log_compile_time") ?as tBool ?? prog.policies.log_compile_time
- var phase_tm = ref_time_ticks()
+ // the plan's fold over the candidates - the first aot-hash per function lands there (cached after)
+ let t_hash = get_time_usec(phase_tm)
+ phase_tm = ref_time_ticks()
var t_declare = 0
var t_probe = 0
var t_irgen = 0
var t_opt = 0
var t_emit = 0
var t_install = 0
- // Content-address the DLL: distinct (AST + codegen version + opt flags) → distinct files, same
- // inputs → same filename → cache hit. Only in dll-mode with the default output path; a pinned
- // jit_output_path or an exe keeps the user's path.
- var dll_hash_basename : string
- if (use_dll && config_out_path |> empty()) {
- dll_hash_basename = jit_dll_basename(funcs, opt_level, size_level, emit_prologue, debug_info, target_triple, jit_split, use_host_cpu)
- output_path = ".jitted_scripts/{prog.thisNamespace}/{dll_hash_basename}"
- }
- // the first aot-hash per function lands here (cached after) — cheap-looking cache hits hide in it
- let t_hash = get_time_usec(phase_tm)
- phase_tm = ref_time_ticks()
// Pass target_triple so init_jit can pin the module data layout
// BEFORE IR construction (wasm32 cross-compile needs 4-byte pointer
// sizes baked into GEP folding from the very first IR instruction).
@@ -935,7 +543,7 @@ def public run_jit(prog : Program?; var ctx : Context?) : bool { // nolint:STYL
// split mode declares per partition inside run_split_codegen (eager-declaring all
// functions into every partition module would be O(funcs x partitions) dead work)
if (!jit_split) {
- var fake_visitor = new LlvmJitVisitor(ctx, g_prim_t, uids, jit_flags, jit_mode, null, attrs)
+ var fake_visitor = new LlvmJitVisitor(prog, ctx, g_prim_t, uids, jit_flags, jit_mode, null, attrs)
ast_gc_guard() {
for (fun in funcs) {
fake_visitor->add_llvm_functions(fun)
@@ -955,7 +563,7 @@ def public run_jit(prog : Program?; var ctx : Context?) : bool { // nolint:STYL
let dll_changed = !(missing |> empty() && disabling |> empty())
recompile_prog = dll_changed
if (recompile_prog) {
- g_dynamic_lib_handle.reset();
+ g_dynamic_lib_handle.reset()
to_log(LOG_INFO, "LLVM JIT: DLL cache miss, codegen for {output_path}.dll\n")
} else {
to_log(LOG_INFO, "LLVM JIT: DLL cache hit {output_path}.dll\n")
@@ -994,11 +602,12 @@ def public run_jit(prog : Program?; var ctx : Context?) : bool { // nolint:STYL
jit_has_externals = irgen_functions(ctx, uids, attrs, funcs, jit_flags, jit_mode, prog)
if (gen_exe) {
let res = inject_main(ctx, g_ctx, prog, exe_main, g_mod, g_prim_t, uids, exe_strict,
- cli_opts.register_all_modules |> unwrap_or(false))
+ plan.register_all_modules)
LINK_WHOLE_LIB = res.link_whole_lib
if (res.fn == null) {
// before finalize_jit, which disposes the engine that owns g_mod
di_finalize()
+ reset_codegen_accumulators()
finalize_jit(use_dll, gen_exe, g_dynamic_lib_handle)
// JIT failed. Couldn't create executable.
// Still return true, codegen ran correctly.
@@ -1073,23 +682,23 @@ def public run_jit(prog : Program?; var ctx : Context?) : bool { // nolint:STYL
}
if (use_dll) {
assert(g_dynamic_lib_handle.handle != null)
- var extern_resolver = new ResolveExternVisitor(g_dynamic_lib_handle, g_prim_t, ctx, clone(uids))
- make_visitor(*extern_resolver) $(adapter_resolve) {
- for (fn in funcs) {
- visit(fn, adapter_resolve)
- }
+ let extern_errors <- resolve_dll_externs(prog, g_dynamic_lib_handle, ctx, uids, funcs)
+ for (e in extern_errors) {
+ failed(e)
}
}
if (!gen_exe && !compile_only) {
- for (fun in funcs) {
- let mangled_name = get_mangled_name(fun)
- let MNH = hash(mangled_name)
- let simfun = get_function_by_mangled_name_hash(MNH, *ctx)
- var gen = generate_llvm_code(g_dynamic_lib_handle, uids.get_dll_fn_name(fun), use_dll)
- if (gen != null) {
- gen |> unsafe(instrument_jit(simfun, fun.at, *ctx))
- } else {
- unsafe(remove_jit(simfun))
+ if (use_dll) {
+ install_dll_functions(g_dynamic_lib_handle, ctx, uids, funcs)
+ } else {
+ for (fun in funcs) {
+ let simfun = get_function_by_mangled_name_hash(hash(get_mangled_name(fun)), *ctx)
+ var gen = generate_llvm_code(g_dynamic_lib_handle, uids.get_dll_fn_name(fun), false)
+ if (gen != null) {
+ gen |> unsafe(instrument_jit(simfun, fun.at, *ctx))
+ } else {
+ unsafe(remove_jit(simfun))
+ }
}
}
}
diff --git a/modules/dasLLVM/daslib/llvm_macro.das b/modules/dasLLVM/daslib/llvm_macro.das
index a2f681c50b..b1782b107d 100644
--- a/modules/dasLLVM/daslib/llvm_macro.das
+++ b/modules/dasLLVM/daslib/llvm_macro.das
@@ -3,41 +3,34 @@ options indenting = 4
options strict_smart_pointers = false
options stack = 4_194_304
+module llvm_macro shared private
+
+//! The emitter's macro context. Nothing requires this module from a program: `llvm_jit_link`
+//! brings it in through `require_module_now` - on a DLL cache miss, and for every artifact
+//! but a cached DLL - and calls the two entries below by name in this context.
+
require llvm/daslib/llvm_boost
require llvm/daslib/llvm_jit_run
-
require daslib/ast_boost
-require daslib/defer
-require jit
-
-module llvm_macro shared private
+[_macro]
+def private llvm_macro_context {
+ pass
+}
-// Lives here (not in llvm_jit_run) so the standalone-exe codegen's skip list
-// in llvm_exe.das — which excludes llvm_func/llvm_macro from the must-be-JIT-able
-// function set — keeps this [no_jit] finalizer out of exe builds.
-// This function should not be calling anything not marked with `no_jit`
-[finalize(late), no_jit]
-def public free_jit_context {
- // Free everything only if it's the last one context.
- if (get_context_share_counter() == 1ul) {
- get_jit_state() $(dll, ee, ctx : void?) {
- if (dll != null) {
- close_dynamic_library(dll)
- } elif (ee != null || ctx != null) {
- assert(ee != null && ctx != null)
- ee |> LLVMRunStaticDestructors()
- LLVMDisposeExecutionEngine(unsafe(reinterpret(ee)))
- LLVMContextDispose(unsafe(reinterpret(ctx)))
- }
- }
+//! the whole codegen pipeline for a simulated program; `ok` is `run_jit`'s answer
+[export]
+def public run_jit_codegen(prog : Program?; var ctx : Context?; var ok : bool?) {
+ unsafe {
+ *ok = run_jit(prog, ctx)
}
}
-[simulate_macro(name="jit_llvm")]
-class JIT_LLVM : AstSimulateMacro {
- def override simulate(prog : Program?; var ctx : Context?) : bool {
- return true if (is_in_completion() || is_compiling_macros() || is_folding())
- return run_jit(prog, ctx)
- }
+//! the in-memory engine's teardown; the finalizer in `llvm_jit_link` delegates here
+[export]
+def public free_jit_engine(ee : void?; ctx : void?) {
+ assert(ee != null && ctx != null)
+ ee |> LLVMRunStaticDestructors()
+ LLVMDisposeExecutionEngine(unsafe(reinterpret(ee)))
+ LLVMContextDispose(unsafe(reinterpret(ctx)))
}
diff --git a/modules/dasLLVM/daslib/llvm_user_modules.das b/modules/dasLLVM/daslib/llvm_user_modules.das
index 1a4671cb0f..52a1554813 100644
--- a/modules/dasLLVM/daslib/llvm_user_modules.das
+++ b/modules/dasLLVM/daslib/llvm_user_modules.das
@@ -7,16 +7,13 @@ module llvm_user_modules shared private
require llvm/daslib/llvm_code_selftest
require llvm/daslib/llvm_tune_fat_word
-require ?dasllama/dasllama_gemm_gen dasllama/dasllama_gemm_register // nolint:STYLE030,LINT019 — used inside the static_if below; on builds without the package both the require and the branch drop (path guard: pure-das package, nothing C++ to guard on)
+require daslib/module_group
+require [llvm_code_generator]
-//! User-extensible wiring for `[llvm_code]` generator modules. The JIT calls
+//! Wiring for `[llvm_code]` generator modules. The JIT calls
//! `register_user_llvm_code_generators` (lazily, in the context that reads the registry)
-//! before the first generator lookup. To plug in a generator module: add a
-//! `require ? ` line above — a PATH guard loads exactly when the
-//! contributor's package is mounted (the guard's own file resolves), so this
-//! module never hard-depends on the contributor. Pair each line with a `static_if
-//! (typeinfo module_exists())` around the registration call below - the
-//! TARGET the guarded require loaded, which is what is visible from here; registered
+//! before the first generator lookup. A generator package joins the `llvm_code_generator`
+//! group from its descriptor and defines `register_llvm_code_generators`. Registered
//! generators are name-keyed and inert for programs whose `[llvm_code]` annotations never
//! name them.
@@ -30,7 +27,5 @@ def public register_user_llvm_code_generators {
g_registered = true
llvm_code_selftest::register_selftest() // worked example + self-test generator (add_i32, add_plus_k)
llvm_tune_fat_word::register_fat_word() // the fat dispatch mask as an LLVM-side word
- static_if (typeinfo module_exists(dasllama_gemm_register)) {
- dasllama_gemm_register::register_dasllama_gemm_code_generators()
- }
+ call_module_group("llvm_code_generator", "register_llvm_code_generators")
}
diff --git a/modules/dasLLVM/examples/hello_jit.das b/modules/dasLLVM/examples/hello_jit.das
index 2c924b9987..c9644387ca 100644
--- a/modules/dasLLVM/examples/hello_jit.das
+++ b/modules/dasLLVM/examples/hello_jit.das
@@ -1,5 +1,5 @@
options gen2
-require llvm/daslib/llvm_macro // This require will jit all functions
+require daslib/just_in_time // the JIT's entry, what -jit requires: every [jit] function is jitted
[jit]
def add(a, b : int) {
diff --git a/modules/dasLLVM/src/dasLLVM.cpp b/modules/dasLLVM/src/dasLLVM.cpp
index 10eb63b578..60ab96c824 100644
--- a/modules/dasLLVM/src/dasLLVM.cpp
+++ b/modules/dasLLVM/src/dasLLVM.cpp
@@ -14,6 +14,7 @@ class Module_LLVM : public Module {
Module_LLVM() : Module("llvm") {
ModuleLibrary lib(this);
lib.addBuiltInModule();
+ registerModuleGroupMember("tune_framework", "llvm/daslib/llvm_tune");
}
virtual ModuleAotType aotRequire ( TextWriter & ) const override {
return ModuleAotType::cpp;
diff --git a/modules/dasLLVM/tests/llvm_jit_link.das b/modules/dasLLVM/tests/llvm_jit_link.das
new file mode 100644
index 0000000000..7735597d1a
--- /dev/null
+++ b/modules/dasLLVM/tests/llvm_jit_link.das
@@ -0,0 +1,130 @@
+options gen2
+require dastest/testing_boost public
+require daslib/fio
+require daslib/strings_boost
+require strings
+require math
+
+// the hit path and its misses: modules/dasLLVM/ARCHITECTURE_JIT_ENTRY.md sec.1-2
+
+def private das_exe() : string {
+ let argv <- get_command_line_arguments()
+ return find(to_generic_path(argv[0]), "/") < 0 ? argv[0] : get_full_file_name(normalize(argv[0]))
+}
+
+def private tail(s : string) : string {
+ return slice(s, max(0, length(s) - 600))
+}
+
+//! the fixture prints its answer and whether the emitter module is in the process; `options
+//! log_compile_time` is what turns the backend's phase line on
+def private fixture(bound : int) : string {
+ return "options gen2\noptions log_compile_time\nrequire daslib/rtti\n\n[export]\ndef main \{\n var s = 0\n for (i in range({bound})) \{\n s += i * i\n \}\n print(\"PROBE \{s\}\\n\")\n print(\"EMITTER \{has_module(\"llvm_macro\")\}\\n\")\n\}\n"
+}
+
+//! a helper the DLL holds; `no_jit` on it after the cold run makes the DLL hold a function the
+//! plan disables, which the probe calls a miss
+def private fixture_with_helper(no_jit_helper : bool) : string {
+ let ann = no_jit_helper ? "[no_jit]\n" : ""
+ return "options gen2\noptions log_compile_time\nrequire daslib/rtti\n\n{ann}def sq(i : int) : int \{\n return i * i\n\}\n\n[export]\ndef main \{\n var s = 0\n for (i in range(10)) \{\n s += sq(i)\n \}\n print(\"PROBE \{s\}\\n\")\n print(\"EMITTER \{has_module(\"llvm_macro\")\}\\n\")\n\}\n"
+}
+
+//! nothing to jit: the only function is `[no_jit]`, so the plan's candidate set is empty
+def private fixture_nothing_to_jit() : string {
+ return "options gen2\noptions log_compile_time\nrequire daslib/rtti\n\n[no_jit, export]\ndef main \{\n print(\"PROBE none\\n\")\n print(\"EMITTER \{has_module(\"llvm_macro\")\}\\n\")\n\}\n"
+}
+
+def private jit_argv(script, cache, pinned : string; in_memory : bool) : array {
+ var argv <- [das_exe(), "-jit", "-log-compile-time", "-module-cache", cache, "-output", pinned, script]
+ if (in_memory) {
+ argv |> push("-jit-no-cache")
+ }
+ return <- argv
+}
+
+def private count_prefix(dir : string; prefix : string) : int {
+ var n = 0
+ fio::dir(dir) $(name) {
+ if (name |> starts_with(prefix)) {
+ n++
+ }
+ }
+ return n
+}
+
+[test]
+def test_jit_link(t : T?) {
+ var terr : string
+ let tmp = create_temp_directory("das_jit_link", terr)
+ if (empty(tmp)) {
+ t |> failure("create_temp_directory: {terr}")
+ return
+ }
+ let script = path_join(tmp, "use_jit.das")
+ let cache = path_join(tmp, "mc.dascache")
+ let pinned = path_join(tmp, "jit_probe")
+ fwrite(script, fixture(10))
+ t |> run("a cold run misses the pinned DLL, brings the emitter in and emits") @(t : T?) {
+ var out = ""
+ let rc = run_and_capture(jit_argv(script, cache, pinned, false), out, 300.0)
+ t |> equal(0, rc, "the fixture runs: {tail(out)}")
+ t |> success(find(out, "PROBE 285") >= 0, "the fixture answered: {tail(out)}")
+ t |> success(find(out, "LLVM JIT: DLL cache miss") >= 0 && find(out, "(codegen") >= 0, "the cold run is a miss that emits: {tail(out)}")
+ t |> success(find(out, "EMITTER true") >= 0, "the emitter module joined the process for the miss: {tail(out)}")
+ t |> equal(0, count_prefix(tmp, "late~"), "the emitter's modules are records of the script's own module cache, not a cache of their own")
+ t |> success(find(out, "LLVM JIT time: plan ") >= 0 && find(out, " emitter load ") >= 0, "the link side's plan and the emitter's load print their times: {tail(out)}")
+ }
+ t |> run("a warm run binds the pinned DLL from the link module alone") @(t : T?) {
+ var out = ""
+ let rc = run_and_capture(jit_argv(script, cache, pinned, false), out, 300.0)
+ t |> equal(0, rc, "the fixture runs: {tail(out)}")
+ t |> success(find(out, "PROBE 285") >= 0, "the fixture answered: {tail(out)}")
+ t |> success(find(out, "LLVM JIT: DLL cache hit") >= 0 && find(out, "(cache") >= 0, "the warm run is a hit: {tail(out)}")
+ t |> success(find(out, "EMITTER false") >= 0, "the emitter module never entered the process on the hit: {tail(out)}")
+ t |> success(find(out, "LLVM JIT time: hash ") >= 0 && find(out, " probe ") >= 0 && find(out, " install ") >= 0, "the hit path prints its phases: {tail(out)}")
+ }
+ t |> run("an in-memory run takes the emitter path and its finalizer frees the engine there") @(t : T?) {
+ var out = ""
+ let rc = run_and_capture(jit_argv(script, cache, pinned, true), out, 300.0)
+ t |> equal(0, rc, "the fixture runs and exits cleanly: {tail(out)}")
+ t |> success(find(out, "PROBE 285") >= 0 && find(out, "EMITTER true") >= 0 && find(out, "(codegen") >= 0, "in-memory codegen ran in the emitter: {tail(out)}")
+ }
+ t |> run("a changed fixture makes the pinned DLL stale, and the miss re-emits") @(t : T?) {
+ fwrite(script, fixture(11))
+ var out = ""
+ let rc = run_and_capture(jit_argv(script, cache, pinned, false), out, 300.0)
+ t |> equal(0, rc, "the fixture runs: {tail(out)}")
+ t |> success(find(out, "PROBE 385") >= 0, "the new body answered: {tail(out)}")
+ t |> success(find(out, "LLVM JIT: DLL cache miss") >= 0 && find(out, "EMITTER true") >= 0, "the stale DLL is a miss that emits: {tail(out)}")
+ }
+ t |> run("a `[no_jit]` function the DLL holds is a miss, and a program with nothing to jit never loads the emitter") @(t : T?) {
+ let helper = path_join(tmp, "use_helper.das")
+ let helperPin = path_join(tmp, "jit_helper")
+ fwrite(helper, fixture_with_helper(false))
+ var out = ""
+ var rc = run_and_capture(jit_argv(helper, path_join(tmp, "mc_helper.dascache"), helperPin, false), out, 300.0)
+ t |> equal(0, rc, "the helper fixture runs: {tail(out)}")
+ t |> success(find(out, "PROBE 285") >= 0 && find(out, "(codegen") >= 0, "the cold run emits the helper: {tail(out)}")
+ fwrite(helper, fixture_with_helper(true))
+ rc = run_and_capture(jit_argv(helper, path_join(tmp, "mc_helper.dascache"), helperPin, false), out, 300.0)
+ t |> equal(0, rc, "the fixture with the disabled helper runs: {tail(out)}")
+ t |> success(find(out, "LLVM JIT: DLL cache miss") >= 0 && find(out, "EMITTER true") >= 0, "the DLL holding a now-disabled function is a miss: {tail(out)}")
+ t |> success(find(out, "LLVM JIT time: plan ") >= 0 && find(out, " probe ") >= 0 && find(out, " emitter load ") >= 0, "the miss prints the link side's plan and probe times with the emitter's load: {tail(out)}")
+ let nothing = path_join(tmp, "use_nothing.das")
+ fwrite(nothing, fixture_nothing_to_jit())
+ rc = run_and_capture(jit_argv(nothing, path_join(tmp, "mc_nothing.dascache"), path_join(tmp, "jit_nothing"), false), out, 300.0)
+ t |> equal(0, rc, "the fixture with nothing to jit runs: {tail(out)}")
+ t |> success(find(out, "PROBE none") >= 0 && find(out, "0 functions to jit") >= 0 && find(out, "EMITTER false") >= 0, "an empty candidate set is answered in the link module, no emitter loaded: {tail(out)}")
+ }
+ t |> run("a pinned DLL another program wrote holds none of this one's functions, and that is a miss") @(t : T?) {
+ let other = path_join(tmp, "use_other.das")
+ fwrite(other, "options gen2\nrequire daslib/rtti\n\n[export]\ndef main \{\n var p = 1\n for (i in range(12)) \{\n p = (p * 3) % 1000\n \}\n print(\"OTHER \{p\}\\n\")\n print(\"EMITTER \{has_module(\"llvm_macro\")\}\\n\")\n\}\n")
+ var out = ""
+ let rc = run_and_capture(jit_argv(other, path_join(tmp, "mc_other.dascache"), pinned, false), out, 300.0)
+ t |> equal(0, rc, "the other program runs: {tail(out)}")
+ t |> success(find(out, "OTHER ") >= 0, "the other program answered: {tail(out)}")
+ t |> success(find(out, "LLVM JIT: DLL cache miss") >= 0 && find(out, "EMITTER true") >= 0 && find(out, "0 functions in") < 0, "the DLL of another program is a miss that emits, not a hit binding nothing: {tail(out)}")
+ }
+ var err : string
+ rmdir_rec(tmp, err)
+}
diff --git a/modules/dasPUGIXML/.das_module b/modules/dasPUGIXML/.das_module
index bac0af7068..f412631e44 100644
--- a/modules/dasPUGIXML/.das_module
+++ b/modules/dasPUGIXML/.das_module
@@ -8,4 +8,7 @@ def initialize(project_path : string) {
}
register_native_path("pugixml", "PUGIXML_boost", "{project_path}/daslib/PUGIXML_boost.das")
register_native_path("pugixml", "linq_fold_xml", "{project_path}/daslib/linq_fold_xml.das")
+ // guarded on the C++ module: the das files sit in every checkout, the module only in a
+ // build configured with it, and linq_fold requires every member
+ register_module_group("linq_fold_source", "pugixml/linq_fold_xml", "pugixml")
}
diff --git a/modules/dasPUGIXML/daslib/linq_fold_xml.das b/modules/dasPUGIXML/daslib/linq_fold_xml.das
index e299fc9f4d..8551063df4 100644
--- a/modules/dasPUGIXML/daslib/linq_fold_xml.das
+++ b/modules/dasPUGIXML/daslib/linq_fold_xml.das
@@ -9,8 +9,7 @@ module linq_fold_xml shared public
//! linq_fold XML source adapter: ``XmlAdapter`` + ``extract_xml_source``. Lets ``_fold`` over a
//! ``from_xml_node(node[, name], type)`` chain emit an inlined DOM child-element walk (mirroring
//! ``PUGIXML_boost``'s generator body) instead of driving the generator + an intermediate array.
-//! Optional companion to ``daslib/linq_fold`` — pulled in via ``require ?pugixml`` and gated by
-//! ``static_if (typeinfo builtin_module_exists(pugixml))``. See daslib/ARCHITECTURE_LINQ.md sec. 37.
+//! Optional companion to ``daslib/linq_fold``. See daslib/ARCHITECTURE_LINQ.md sec. 37.
require pugixml public
require pugixml/PUGIXML_boost // nolint:STYLE030 — build_xml_row / read_xml_field / peek_xml_field are emitted via qmacro, invisible to the require-usage scan
@@ -531,3 +530,19 @@ def extract_xml_source(var top : Expression?) : ExprCall? {
if (na != 2 && na != 3) return null
return c
}
+
+[macro_function]
+def private recognize_xml_source(var top : Expression?; at : LineInfo) : LinqFoldSourceMatch {
+ var xmlCall = extract_xml_source(top)
+ if (xmlCall == null) return LinqFoldSourceMatch()
+ var topClone = clone_expression(top)
+ var nameArg : Expression? = length(xmlCall.arguments) == 3 ? clone_expression(xmlCall.arguments[1]) : null
+ return LinqFoldSourceMatch(matchTop = topClone, ctxTop = topClone,
+ adapter = new XmlAdapter(nodeExpr = clone_expression(xmlCall.arguments[0]), srcName = qn("xnode", at),
+ elemType = clone_type(xmlCall._type.firstType), nameArg = nameArg))
+}
+
+def register_linq_fold_source {
+ //! The ``linq_fold_source`` group entry: registers the XML recognizer in the current macro context.
+ add_linq_fold_source(@@ < LinqFoldSourceRecognizer > recognize_xml_source)
+}
diff --git a/modules/dasPUGIXML/src/dasPUGIXML.cpp b/modules/dasPUGIXML/src/dasPUGIXML.cpp
index 8ea2b9f4eb..a3ea683beb 100644
--- a/modules/dasPUGIXML/src/dasPUGIXML.cpp
+++ b/modules/dasPUGIXML/src/dasPUGIXML.cpp
@@ -704,6 +704,7 @@ pugi::xpath_query * pugiXPathCompileWithVars ( char * query, pugi::xpath_variabl
class Module_PUGIXML : public Module {
public:
Module_PUGIXML() : Module("pugixml") {
+ registerModuleGroupMember("linq_fold_source", "pugixml/linq_fold_xml");
ModuleLibrary lib;
lib.addModule(this);
lib.addBuiltInModule();
diff --git a/modules/dasSQLITE/.das_module b/modules/dasSQLITE/.das_module
index 244a29199b..4b74d5f393 100644
--- a/modules/dasSQLITE/.das_module
+++ b/modules/dasSQLITE/.das_module
@@ -10,4 +10,7 @@ def initialize(project_path : string) {
for (path in sqlite_paths) {
register_native_path("sqlite", "{path}", "{project_path}/daslib/{path}.das")
}
+ // guarded on the C++ module: the descriptor and the das files sit in every checkout, the
+ // module only in a build configured with it, and sql_boost requires every member
+ register_module_group("sql_provider", "sqlite/sqlite_provider", "sqlite")
}
diff --git a/modules/dasSQLITE/PROVIDER_CONTRACT.md b/modules/dasSQLITE/PROVIDER_CONTRACT.md
index 97eb734802..a5d31eb54f 100644
--- a/modules/dasSQLITE/PROVIDER_CONTRACT.md
+++ b/modules/dasSQLITE/PROVIDER_CONTRACT.md
@@ -65,13 +65,17 @@ Registries live per macro context (consumer-requires-contributor; see
skills/das_macros.md "Macro modules each compile into their own context"), so
registration is pulled, not pushed: each consulting macro module (`daslib/sql_boost`,
`daslib/sql_linq`) has a `[_macro]` twin calling `sql_register_present_providers`
-(in `daslib/sql_boost`), which optionally-requires each provider's **registration
-shim** (`require ?sqlite sqlite/sqlite_provider`) and calls its registration
-function under `static_if (typeinfo builtin_module_exists())`. A new provider
-therefore ships a lean shim module (stmt factories + dialect hooks + caps; it must
-NOT require the provider's boost - that would cycle through `daslib/sql_boost`) and
-adds one `require ?` line plus one `static_if` branch to
-`sql_register_present_providers`. Entry fields:
+(in `daslib/sql_boost`), which requires the `sql_provider` **group** and calls
+`register_provider` on every member (`call_module_group`, `daslib/module_group`). A
+new provider therefore ships a lean **registration shim** module (stmt factories +
+dialect hooks + caps; it must NOT require the provider's boost - that would cycle
+through `daslib/sql_boost`) that defines `register_provider`, and joins the group:
+`register_module_group("sql_provider", "/_provider", "")` in its
+`.das_module` descriptor - the third argument guards the row on the C++ module, since the
+descriptor and the das files sit in every checkout and the module only in a build configured
+with it - plus `registerModuleGroupMember` with the first two strings in the C++ module's
+`initMain` (the binder generates the constructor) for a host that runs no descriptors.
+Nothing in `daslib/` is edited. Entry fields:
| Field | SQLite value | Consulted by |
|---|---|---|
diff --git a/modules/dasSQLITE/daslib/sqlite_provider.das b/modules/dasSQLITE/daslib/sqlite_provider.das
index 060ae041e9..a424173387 100644
--- a/modules/dasSQLITE/daslib/sqlite_provider.das
+++ b/modules/dasSQLITE/daslib/sqlite_provider.das
@@ -18,9 +18,9 @@ require daslib/templates_boost
//! SQLite's `SqlProviderInfo` registration shim — the lean bridge between the raw
//! `sqlite` C binding and the neutral registry in `daslib/sql_provider`. Deliberately
-//! does NOT require `sqlite/sqlite_boost`: `daslib/sql_provider` pulls this file into
-//! every consulting macro context via `require ?sqlite`, and requiring the full boost
-//! here would cycle (sqlite_boost → daslib/sql_boost → daslib/sql_provider → here).
+//! does NOT require `sqlite/sqlite_boost`: `daslib/sql_boost` pulls this file into
+//! every consulting macro context as a member of the `sql_provider` group, and requiring
+//! the full boost here would cycle (sqlite_boost → daslib/sql_boost → here).
//! Runtime surface (SqlRunner, exec/query, runner helpers) lives in `sqlite/sqlite_boost`.
def sqlite3_column_text_(stmt : sqlite3_stmt?; col : int) : string {
@@ -122,10 +122,8 @@ def private _sqlite_render_json_extract(target : string; jsonPath : string) : st
def private _sqlite_quote_id(id : string) : string => sql_quote_id(id)
-def register_sqlite_provider {
- //! Registers the sqlite `SqlProviderInfo` entry in the CURRENT macro context.
- //! Reached via `sql_register_present_providers` (daslib/sql_provider) from each
- //! consulting macro module's own `[_macro]`.
+def register_provider {
+ //! The `sql_provider` group entry: registers the sqlite `SqlProviderInfo` in the CURRENT macro context.
var info = SqlProviderInfo(
name = "sqlite",
runnerModule = "sqlite_boost",
diff --git a/modules/dasSQLITE/src/dasSQLITE.main.cpp b/modules/dasSQLITE/src/dasSQLITE.main.cpp
index 9ba83b6371..77d54ff4e7 100644
--- a/modules/dasSQLITE/src/dasSQLITE.main.cpp
+++ b/modules/dasSQLITE/src/dasSQLITE.main.cpp
@@ -80,6 +80,7 @@ int sqlite3_open_v2_no_vfs ( const char * filename, sqlite3 ** ppDb, int flags )
}
void Module_dasSQLITE::initMain() {
+ registerModuleGroupMember("sql_provider", "sqlite/sqlite_provider");
addExtern(*this,lib,"sqlite3_exec",
SideEffects::worstDefault, "sqlite3_exec")
diff --git a/modules/dasUnitTest/unit_test.das.inc b/modules/dasUnitTest/unit_test.das.inc
index 9a909d6317..9db5163853 100644
--- a/modules/dasUnitTest/unit_test.das.inc
+++ b/modules/dasUnitTest/unit_test.das.inc
@@ -1,32 +1,32 @@
-static unsigned char unit_test_das[] = {
-0x6f,0x70,0x74,0x69,0x6f,0x6e,0x73,0x20,
-0x67,0x65,0x6e,0x32,0x0a,
-0x6f,0x70,0x74,0x69,0x6f,0x6e,0x73,0x20,
-0x72,0x65,0x6d,0x6f,0x76,0x65,0x5f,0x75,
-0x6e,0x75,0x73,0x65,0x64,0x5f,0x73,0x79,
-0x6d,0x62,0x6f,0x6c,0x73,0x20,0x3d,0x20,
-0x66,0x61,0x6c,0x73,0x65,0x0a,
-0x0a,
-0x72,0x65,0x71,0x75,0x69,0x72,0x65,0x20,
-0x55,0x6e,0x69,0x74,0x54,0x65,0x73,0x74,
-0x0a,
-0x0a,
-0x6c,0x65,0x74,0x20,0x4f,0x54,0x48,0x45,
-0x52,0x5f,0x55,0x4e,0x49,0x54,0x5f,0x54,
-0x45,0x53,0x54,0x5f,0x43,0x4f,0x4e,0x53,
-0x54,0x41,0x4e,0x54,0x20,0x3d,0x20,0x55,
-0x4e,0x49,0x54,0x5f,0x54,0x45,0x53,0x54,
-0x5f,0x43,0x4f,0x4e,0x53,0x54,0x41,0x4e,
-0x54,0x20,0x2b,0x20,0x31,0x3b,0x0a,
-0x0a,
-0x64,0x65,0x66,0x20,0x73,0x74,0x61,0x72,
-0x74,0x5f,0x65,0x66,0x66,0x65,0x63,0x74,
-0x28,0x6e,0x61,0x6d,0x65,0x3b,0x20,0x74,
-0x6d,0x29,0x20,0x7b,0x0a,
-0x20,0x20,0x20,0x20,0x72,0x65,0x74,0x75,
-0x72,0x6e,0x20,0x73,0x74,0x61,0x72,0x74,
-0x5f,0x65,0x66,0x66,0x65,0x63,0x74,0x28,
-0x6e,0x61,0x6d,0x65,0x2c,0x20,0x74,0x6d,
-0x2c,0x20,0x31,0x2e,0x30,0x29,0x0a,
-0x7d,0x0a,
-};
+static unsigned char unit_test_das[] = {
+0x6f,0x70,0x74,0x69,0x6f,0x6e,0x73,0x20,
+0x67,0x65,0x6e,0x32,0x0a,
+0x6f,0x70,0x74,0x69,0x6f,0x6e,0x73,0x20,
+0x72,0x65,0x6d,0x6f,0x76,0x65,0x5f,0x75,
+0x6e,0x75,0x73,0x65,0x64,0x5f,0x73,0x79,
+0x6d,0x62,0x6f,0x6c,0x73,0x20,0x3d,0x20,
+0x66,0x61,0x6c,0x73,0x65,0x0a,
+0x0a,
+0x72,0x65,0x71,0x75,0x69,0x72,0x65,0x20,
+0x55,0x6e,0x69,0x74,0x54,0x65,0x73,0x74,
+0x0a,
+0x0a,
+0x6c,0x65,0x74,0x20,0x4f,0x54,0x48,0x45,
+0x52,0x5f,0x55,0x4e,0x49,0x54,0x5f,0x54,
+0x45,0x53,0x54,0x5f,0x43,0x4f,0x4e,0x53,
+0x54,0x41,0x4e,0x54,0x20,0x3d,0x20,0x55,
+0x4e,0x49,0x54,0x5f,0x54,0x45,0x53,0x54,
+0x5f,0x43,0x4f,0x4e,0x53,0x54,0x41,0x4e,
+0x54,0x20,0x2b,0x20,0x31,0x3b,0x0a,
+0x0a,
+0x64,0x65,0x66,0x20,0x73,0x74,0x61,0x72,
+0x74,0x5f,0x65,0x66,0x66,0x65,0x63,0x74,
+0x28,0x6e,0x61,0x6d,0x65,0x3b,0x20,0x74,
+0x6d,0x29,0x20,0x7b,0x0a,
+0x20,0x20,0x20,0x20,0x72,0x65,0x74,0x75,
+0x72,0x6e,0x20,0x73,0x74,0x61,0x72,0x74,
+0x5f,0x65,0x66,0x66,0x65,0x63,0x74,0x28,
+0x6e,0x61,0x6d,0x65,0x2c,0x20,0x74,0x6d,
+0x2c,0x20,0x31,0x2e,0x30,0x29,0x0a,
+0x7d,0x0a,
+};
diff --git a/plans/jit_compile_time.md b/plans/jit_compile_time.md
index 9e9406ac54..e4bf693f26 100644
--- a/plans/jit_compile_time.md
+++ b/plans/jit_compile_time.md
@@ -60,3 +60,58 @@ that stay:
4. **Macro-emitter changes are invisible to every cache key** (QUIRK 21; QUIRK 15's unquirk
note wants the generator hash folded into the sidecar identity too). Done = an emitter
edit invalidates exactly the families it generates.
+
+## Front-end record read - the macro-context simulate share (ledgered 2026-09-08)
+
+A warm `-jit` hello world spends 70 of its 88 ms in the compile, and the compile is the
+module-cache read: 61 records deserialized (~10 ms of raw bytes) plus the macro-context
+simulate of every macro module the program requires - typemacro_boost 4.6 ms, clargs 4.1,
+templates_boost 3.5, quote 2.8, ast_boost 2.5, llvm_jit_link 4.7, some twenty modules in all
+(`-log-compile-time`, the `cache read took ... -- macro simulate` lines). The emitter's own
+context no longer loads on a DLL hit (`llvm_jit_link`), so what remains is daslib's. Separate
+from the JIT and from the group/late-require work; not started. Also in the read:
+`bindings/llvm_func` re-parses on every run (11 ms) - its `require dasbind` drifts the
+builtin hash, so the record never serves ("reparsing in place").
+
+Levers, unmeasured: a macro context that simulates lazily on the first macro call instead of
+at record read; a cheaper simulate for a context whose program did not change (the record could
+carry the simulated context's tables); a `[_macro]`-free module skipping the context entirely.
+
+## Follow-ups from the require-group review (ledgered 2026-09-08)
+
+- **A changed record resumes instead of cutting.** `trySerializeProgramModule` cuts the stream
+ at a changed file ("file changed"), so every record after it - the emitter's ~40 modules a
+ JIT miss appended to the script's stream - parses from source on the run after an edit to
+ the script. The record header carries the payload size, so a changed file could be reparsed
+ in place and the rest served, as the corrupt-record resume already does; the reason it cuts
+ today is that a later module may depend on the changed one through a macro. A resume gated
+ on "no later record lists this file among its macro dependencies" keeps the emitter warm.
+- **`collectRequireNames` scans the source twice more per module** - the reader recomputes the
+ collector's answer for every record whose hash matched, the writer for every fresh parse -
+ and the reader's copy evaluates the guards (a deferred module load inside a cache read). The
+ walk already ran the collector over the same file; the answer could ride the `ModuleInfo`.
+- **`require_module_now` records ride the script's stream**, so a script that ever JIT-missed
+ carries the emitter's records (~34 MB) in its own cache under the default-directory LRU. One
+ emitter stream shared by every script would need a stream the environment can bind beside
+ the host's, which is the persistent-serializer shape a host installs, not the CLI's.
+- **The hit path binds `llvm/bindings/llvm_func`** (2320 `[extern]` lines, ~10 ms of the warm
+ 0.08 s) for three host queries - triple, CPU name, CPU features - in `jit_env_salt`. Three
+ C++ binds in the `jit` module would drop the binding from the hit path.
+- **The extern resolver looks every operator and intrinsic node up in the DLL** - the lowered
+ ones answer "no such slot" after a mangled-name build and a `dlsym` miss - where the emitter's
+ visitor skipped them by `has_intrinsic` / `isExprOp2_Func` before any of that.
+- **The in-memory engine's finalizer runs through the emitter's macro context**
+ (`free_jit_context`'s engine branch), so a host whose context outlives module shutdown reaches
+ a context that is gone; a C++ `free_jit_engine` in the `jit` module would not.
+- **A late require nested in a parse or a record read is never cached.** `requireModuleNow`
+ hides the stream (`LateRequireEnvScope::hideStream`) when the bound program is compiling or
+ the reader is inside a record, so a `[call_macro]`'s or a macro module's `[init]`'s late
+ require parses from source on every run. The JIT's simulate-time require, the one that
+ matters today, is between records and joins the stream; the nested shapes would need a
+ reader that can suspend a record and a writer that can insert after the enclosing module's.
+- **Nothing checks that a descriptor's `grp` rows agree with the module's C++ rows.** A
+ static host gets the rows `registerModuleGroupMember` adds from the module constructor, a
+ shared_module build the rows the descriptor's `register_module_group` recorded; the two are
+ spelled by hand in two files (`dasLLVM.cpp` / `.das_module`, and the sibling pairs) and drift
+ is a member one build has and the other lacks. A gate that loads each descriptor and diffs
+ its `grp` rows against the constructor's registry would catch it.
diff --git a/skills/comment_style_hygiene.md b/skills/comment_style_hygiene.md
index 5c43450961..ed51020456 100644
--- a/skills/comment_style_hygiene.md
+++ b/skills/comment_style_hygiene.md
@@ -71,7 +71,10 @@ cannot tell the two apart, so this boundary is the reviewer's to hold.
it in a name, a shape, or a test, or documents itself in the module's `.md`. Kept when one
is earned: `//!` and `/** */` docs on a contract a CALLER must know - public API always, a
non-public helper when callers inside the module need the contract - `NOLINT` /
-`clang-format` suppressions, the leading header block. Existing comments answer to the
+`clang-format` suppressions, the leading header block, and the bare section pointer
+`// /ARCHITECTURE.md sec.N` on the function that implements the section (C++ has no
+`[arch]` annotation; the pointer is its citation, and nothing checks it - a renumbered section
+orphans every pointer to it). Existing comments answer to the
deletion test; editing one so it stays true is not a new comment, and the edited comment
answers to that test as a whole.
diff --git a/skills/das_macros.md b/skills/das_macros.md
index 809942d18d..45cdce164c 100644
--- a/skills/das_macros.md
+++ b/skills/das_macros.md
@@ -64,6 +64,19 @@ dispatcher branch naming the contributor's symbols - `typeinfo module_exists(isUsed(fn)` / `program->indexOf(fn)` /
+ `program->isJitSelected(fn)` (C++), `is_used(prog, fn)` / `function_index(prog, fn)` /
+ `variable_index(prog, var)` / `is_jit_selected(prog, fn)` (das) - a shared module's objects
+ serve every program in the process (`src/ast/ARCHITECTURE.md` sec.4). `func.moreFlags.requestJit`
+ is the `[jit]` annotation, not the selection
## Diffing interpreter against AOT
diff --git a/skills/linq_fold_patterns.md b/skills/linq_fold_patterns.md
index da0dd3bbe8..67c648272c 100644
--- a/skills/linq_fold_patterns.md
+++ b/skills/linq_fold_patterns.md
@@ -141,7 +141,7 @@ what would otherwise be many lookalike chains:
- Runtime component-name list form. Same decs splices as the template form.
* - `unsafe(from_xml_node(node[, name], type))`
- `extract_xml_source` (`XmlAdapter`, `modules/dasPUGIXML/daslib/linq_fold_xml.das`)
- - Optional source - only when the `pugixml` module is linked (`require ?pugixml` + `static_if (typeinfo builtin_module_exists(pugixml))`). Emits an inlined DOM child-element walk replacing the generator, and **field-prunes** the per-element materialization (pass 2b): the chain body is scanned for the `Row` fields it reads, and only those attributes are read via `read_xml_field` into scalar locals - unread fields (notably `string` fields, whose `clone_string` is the alloc cost) are never touched, so a float-only chain runs alloc-free and JIT beats the equivalent SQLite query. A whole-row escape (`to_array` / identity `_select(_)` / pass-to-fn) routes to the full `build_xml_row` instead. The `XmlAdapter` **rides every pattern row** (`try_splice_patterns` runs with no `onlyRow` restriction); per-row `requires` predicates and the adapter's capability hooks (`can_join` / `can_group_by` / `defers_materialization` / the `non_array_source` gate) decide what fuses, and a shape it can't fuse cascades to tier-2 - see [linq_fold_xml_patterns](https://daslang.io/doc/reference/linq_fold_patterns.html#linq-fold-xml-patterns) for the full fuse/defer breakdown. `unsafe` is required (the source is `[unsafe_outside_of_for]`) and the node is passed by value (`var root` - `_fold`'s macro-arg inference skips the const&->value copy).
+ - Optional source - present when the `pugixml` module is in the tree (`linq_fold_xml` joins the `linq_fold_source` group from pugixml's descriptor; `daslib/linq_fold` calls every member's `register_linq_fold_source`). Emits an inlined DOM child-element walk replacing the generator, and **field-prunes** the per-element materialization (pass 2b): the chain body is scanned for the `Row` fields it reads, and only those attributes are read via `read_xml_field` into scalar locals - unread fields (notably `string` fields, whose `clone_string` is the alloc cost) are never touched, so a float-only chain runs alloc-free and JIT beats the equivalent SQLite query. A whole-row escape (`to_array` / identity `_select(_)` / pass-to-fn) routes to the full `build_xml_row` instead. The `XmlAdapter` **rides every pattern row** (`try_splice_patterns` runs with no `onlyRow` restriction); per-row `requires` predicates and the adapter's capability hooks (`can_join` / `can_group_by` / `defers_materialization` / the `non_array_source` gate) decide what fuses, and a shape it can't fuse cascades to tier-2 - see [linq_fold_xml_patterns](https://daslang.io/doc/reference/linq_fold_patterns.html#linq-fold-xml-patterns) for the full fuse/defer breakdown. `unsafe` is required (the source is `[unsafe_outside_of_for]`) and the node is passed by value (`var root` - `_fold`'s macro-arg inference skips the const&->value copy).
* - `unsafe(each_kv(tab))` / `keys(tab)` / `values(tab)`
- `extract_table_source` (`TableAdapter`, `daslib/linq_fold_table.das`)
- In-tree source - recognized by builtin-module identity (`match_call_in_module`) **plus** a table-typed argument (`table` / `table`), so a user `keys`/`values`/`each_kv` overload never fires it. The kv lane (`each_kv`) binds `kv.key` / `kv.value` and **usage-prunes the walk**: a chain touching only `.value` walks `values(tab)` alone, only `.key` (or neither) walks `keys(tab)` alone - half the slot-skip work of the zipped two-iterator form, which is emitted only when both sides (or the whole pair) are read. A whole-pair escape binds a named-tuple copy, so the kv lane fuses **copyable value types only** - a non-copyable-valued `each_kv` falls through and the surviving instantiation concept-asserts (error 31400; the keys/values lanes still fuse such tables). Bare `count()` / `long_count()` folds to O(1) `length(tab)`; a plain `distinct` over raw keys/kv elements is **dropped** before matching (keys are unique by construction; only uniqueness-preserving prefix ops allow the drop - a preceding `select` keeps the distinct, and the values lane always keeps it). **Point-lookup folds** (`try_table_point_lookup`): a key-equality `where` (`kv.key == X`, bare `k == X` on the keys lane, either operand order; predicate-form `any(p)` / `count(p)` too) against a loop-invariant, side-effect-free `X` folds the whole walk to an O(1) probe - `any` / keys-lane `contains(X)` -> `key_exists(tab, X)`, `count` -> `key_exists ? 1 : 0`, `first` / `first_or_default` (+/- one trailing `select`) -> a `tab?[X]` probe with the scan's exact semantics (panic on a missing `first`, eagerly-bound default value otherwise). The key-equality may carry **residual conjuncts** - `kv.key == X && residual...` with the key-equality as the *leftmost* conjunct (a leading run of consecutive `where` calls AND-merges first, so `where(key == X) |> where(res)` is the same shape): the probe evaluates the residual on the probed element only, a false residual routes to the same miss path, and no purity gate is needed on the residual because keys are unique - under `&&` short-circuit it runs at most once on both paths. Anything else - a compound predicate whose leftmost conjunct is not the key-equality, other comparison operators, an `X` that reads the binder or has side effects (the scan evaluates `X` per element, the probe once) - keeps the scan. `order_by` / `take` / `first` observe the table's unspecified slot order, exactly like a hand `for (k, v in keys(t), values(t))` loop. **Joins fuse on either side** (`can_join` is on; the adapter rides the shared `emit_array_join` through its own `wrap_source_loop`): a table *lead* walks its pruned slot iterator(s) as the probe loop; a table in the *srcB slot* joined on its bare key - `d.key` on the kv lane, the bare element on a `keys(set)` source - skips the join's internal `table>` entirely and probes the user's table per lead row (`join_keyb_is_bare_key` + `build_join_probe_pieces`; unique table keys make the probe == hash semantics exactly). The probe is itself usage-pruned: count-no-where and key-only shapes stay on `key_exists`, value shapes bind the matched value **by reference** from a `tab?[k]` pointer (no copy), and only a whole-pair use binds the kv tuple. A non-bare b-key keeps the hashed build over the kv iterator; `group_join` (outer - its result consumes the whole bucket) always keeps it. **`group_by` fuses** (`can_group_by` is on; `build_group_by_adapter` hands `plan_group_by_core` a fresh `TableAdapter`, so the bucket-fill loop is the usage-pruned slot walk - a group key over `kv.value.brand` walks `values(tab)` alone) for the plain-lead shape only: `join |> group_by` over a table lead declines (the upstream-join arm returns null) and reverse has no backward slot walk - those shapes cascade to tier-2 (see `history/linq_fold/LINQ_TO_TABLE.md`). **`to_table()` sinks fuse** (table-buffer materializer row above): the chain inserts straight into the result table - a bare `each_kv(tab).to_table()` is a reserve-ahead table clone through the fused walk, and a `keys(tab)` chain lands in the `table` set form. `unsafe` is required at an unfused chain head (the sources are `[unsafe_outside_of_for]`); fused chains rewrite the head before inference.
diff --git a/skills/tune.md b/skills/tune.md
index 4295edb923..9afb1bf3e8 100644
--- a/skills/tune.md
+++ b/skills/tune.md
@@ -10,7 +10,7 @@ other tier the reference body runs verbatim.
`daslib/tune` registers all six annotations - `[tune]`, `[tune_perm]`,
`[tune_companion]`, `[tune_scope]`, `[tune_policy]`, `[llvm_code]` - and the
default-policy pass; the framework behind them is `llvm/daslib/llvm_tune`,
-which `daslib/tune` reaches through the `?llvm` witness guard (present exactly when the build is configured with dasLLVM) and re-exports; a direct `require llvm/daslib/llvm_tune` marks a program as framework-only - without dasLLVM its files still compile, but the shells are inert and no `_variants()` registry exists. **Any program
+which `daslib/tune` reaches as a member of the `tune_framework` group (dasLLVM joins it exactly when the build is configured with it) and re-exports; a direct `require llvm/daslib/llvm_tune` marks a program as framework-only - without dasLLVM its files still compile, but the shells are inert and no `_variants()` registry exists. **Any program
that reaches the framework through `daslib/tune` alone can spell the annotations and compile whether or not dasLLVM is configured.**
Without dasLLVM every one of them does nothing at all - no sidecar read, no
stamp, no `_variants()` registry, no tuner spawn - and the reference
diff --git a/skills/xml.md b/skills/xml.md
index 6b5e9f25dd..1a1383b99e 100644
--- a/skills/xml.md
+++ b/skills/xml.md
@@ -85,7 +85,7 @@ let cars <- unsafe(from_xml_node(root, "car", type)) |> to_array() // tag
- **Defaults:** a missing attribute leaves the field at its declared default (`year : int = 2000` above), because the field value is passed as the accessor's fallback.
- **Lifetime-safe:** rows are owned values - string fields are cloned out of the document - so results collected with `to_array` / a comprehension stay valid **past** the `parse_xml` / `open_xml` RAII block. (Contrast: a raw `xml_node` must not escape the block.)
- **`unsafe` outside a `for`:** `from_xml_node` is `[unsafe_outside_of_for]`. A `for` loop and a comprehension are safe; piping it into `to_array` / `linq_boost` outside a `for` needs `unsafe(...)` **around the source call only** - expression-form `unsafe` does not reach into nested call arguments, so `unsafe(from_xml_node(...) |> to_array())` still fails at the inner call.
-- **Fused `_fold` lane (pass 2a/2b):** `_fold(unsafe(from_xml_node(root, type))._where(...)._select(...).count()/sum()/min()/max()/average()/any()/first()/take(N)...)` emits an inlined DOM child-walk via `XmlAdapter` (`require ?pugixml pugixml/linq_fold_xml`, gated by `static_if (typeinfo builtin_module_exists(pugixml))` in `daslib/linq_fold`) - no generator, no intermediate array. Pass the node by value (`var root`, not `let root`) since `_fold`'s macro-arg inference skips the const&->value copy. **Field-pruning (pass 2b):** the walk reads only the `Row` fields the chain actually references (via `read_xml_field` into scalar locals) - unread fields, especially `string` fields whose `clone_string` is the alloc cost, are never touched, so a float-only chain is alloc-free and JIT beats the equivalent SQLite query. A whole-row escape (`to_array` / identity `_select(_)` / pass-to-fn) falls back to the full `build_xml_row`. Cascade shapes (order_by / distinct / group_by / join) fuse via the cascade arms (`wrap_source_loop` / `build_group_by_adapter`). **Reverse** has backward-DOM fast paths: `reverse |> take(N)` and no-predicate `last()` / `reverse |> first` walk `last_child` / `previous_sibling`, visiting only the kept tail (reverse-take m5f went 88.9 -> 0.0 ns/op) - while predicated `[where] |> last` deliberately stays on the forward walk, since reverse DOM traversal is ~2x cache-hostile per node and a deep match would regress. All shapes produce the same result as plain array linq (a full parity suite under `tests/dasPUGIXML/parity_xml_*.das`, repo-only, asserts `m5f == m3f == m3` across the whole `loop_or_count` + cascade surface). A `from_xml_node` source can also be **joined / union'd with an in-memory array** directly (`from_xml_node(...) |> _join(dealersArray, ...)`) via linq's mixed `(iterator, array)` overloads. See [tutorials/dasPUGIXML/05_linq_over_xml.das](tutorials/dasPUGIXML/05_linq_over_xml.das).
+- **Fused `_fold` lane (pass 2a/2b):** `_fold(unsafe(from_xml_node(root, type))._where(...)._select(...).count()/sum()/min()/max()/average()/any()/first()/take(N)...)` emits an inlined DOM child-walk via `XmlAdapter` (`linq_fold_xml` joins the `linq_fold_source` group from pugixml's descriptor; `daslib/linq_fold` calls every member's `register_linq_fold_source`) - no generator, no intermediate array. Pass the node by value (`var root`, not `let root`) since `_fold`'s macro-arg inference skips the const&->value copy. **Field-pruning (pass 2b):** the walk reads only the `Row` fields the chain actually references (via `read_xml_field` into scalar locals) - unread fields, especially `string` fields whose `clone_string` is the alloc cost, are never touched, so a float-only chain is alloc-free and JIT beats the equivalent SQLite query. A whole-row escape (`to_array` / identity `_select(_)` / pass-to-fn) falls back to the full `build_xml_row`. Cascade shapes (order_by / distinct / group_by / join) fuse via the cascade arms (`wrap_source_loop` / `build_group_by_adapter`). **Reverse** has backward-DOM fast paths: `reverse |> take(N)` and no-predicate `last()` / `reverse |> first` walk `last_child` / `previous_sibling`, visiting only the kept tail (reverse-take m5f went 88.9 -> 0.0 ns/op) - while predicated `[where] |> last` deliberately stays on the forward walk, since reverse DOM traversal is ~2x cache-hostile per node and a deep match would regress. All shapes produce the same result as plain array linq (a full parity suite under `tests/dasPUGIXML/parity_xml_*.das`, repo-only, asserts `m5f == m3f == m3` across the whole `loop_or_count` + cascade surface). A `from_xml_node` source can also be **joined / union'd with an in-memory array** directly (`from_xml_node(...) |> _join(dealersArray, ...)`) via linq's mixed `(iterator, array)` overloads. See [tutorials/dasPUGIXML/05_linq_over_xml.das](tutorials/dasPUGIXML/05_linq_over_xml.das).
## Quick accessors (with defaults)
diff --git a/src/ast/ARCHITECTURE.md b/src/ast/ARCHITECTURE.md
index d9ef4373ee..11f0772024 100644
--- a/src/ast/ARCHITECTURE.md
+++ b/src/ast/ARCHITECTURE.md
@@ -27,7 +27,15 @@ the `CodeOfPolicies` of the compile that wrote it, and a record whose policies d
reading compile's is never served - a lint compile and a run compile of one graph never share a
record. The mismatch fails the record the way damage does, reparsed in place rather than cutting
the stream (damage landing on the policy bytes must not cut it either), and the writeback that
-repairs it carries the new policies. The first mismatch is the cutoff: the reader marks the stream failed, the
+repairs it carries the new policies. The record header also carries the text collector's
+answer for the file when the record was written (`collectRequireNames` over `getAllRequire`,
+guards and groups already applied, names sorted), and the reader runs the same collector over the
+file it is about to serve and compares: a member a group gained or a guard a build flipped changes
+a module's dependencies without touching its bytes, and a record served then would carry the old
+dependency set. Both sides are one function, so a require only the walk knows - a host's
+`parseCustomRequire` - compares equal to itself instead of cutting the stream on every warm read.
+That mismatch is a cutoff (`require set changed`), since the modules after it may have moved too.
+The first mismatch is the cutoff: the reader marks the stream failed, the
module and everything after it parse from source, and the writer rewrites the whole file - the
served records re-serialized from the modules the reader restored, then the freshly parsed
ones. A record whose header matched but whose payload fails to deserialize reparses in place,
@@ -110,7 +118,14 @@ require guard (`require ?mod`) and `builtin_module_exists` ask whether the build
module (`guardModuleAvailable`): linked in, or waiting in a manifest row, which the guard
loads then - so `require ?das_metal metal/das_metal_boost` still means "on a build with
Metal", a cold start and a warm start answer alike, and `llvm`, a witness module no das file
-requires unguarded, comes in through the guards `daslib/tune` places on it. A load adds
+requires unguarded, comes in through the guards dasLLAMA places on it. A guard whose name
+holds a `/` is a path instead: the guard is taken when the guard's own file resolves through the
+compile's `FileAccess` - the rail for a pure-das package, which has nothing C++ to guard on, and
+for a witness of a cross-package dependency. A guard the build does not have skips the require
+silently and without resolving the target, so a skipped require probes no file paths; a guard the
+build has over a target that does not resolve is the ordinary missing-module error. Neither form
+falls back to the target's own resolvability: a module's source directory sits in every checkout
+whatever the build configured. A load adds
nothing to `$`: a module-cache record carries each builtin module's cumulative hash of
mangled names, and a process that loaded a different set of C++ modules would otherwise fail
every record on `$`, so a `vector` of a handled element registers into the element's
@@ -123,7 +138,29 @@ that names one compares the part after `::` (`daslib/ast_boost`'s printer). `-ig
reads and writes no manifest: every descriptor compiles and every C++ module loads on start,
the form a tool that enumerates modules - the MCP server - runs under.
`no_manifest()` inside `initialize` marks the descriptor as one that runs on every start: its
-manifest carries the stamp and the flag and no rows, and is not rewritten. With
+manifest carries the stamp and the flag and no rows, and is not rewritten. A module group is a
+name that `require [group]` expands to a list: `register_module_group(group, member, guard)` in
+a descriptor - recorded as a `grp` row and replayed - or `registerModuleGroupMember` from a C++
+module adds a member, once (`ast_module.cpp`, one process-wide registry under a mutex, kept for
+the life of the process like the native paths: a descriptor registers once per process, so a
+cleared registry could only lose members); the list comes back sorted by member path, since the
+scan registers in `readdir` order, which no platform promises. A member's guard is the guard a
+hand-written `require ?guard member` would carry - a module name, or a path when it holds a
+`/` - and is evaluated wherever the list is read; a descriptor whose das files sit in every
+checkout guards its row on the C++ module only a build configured with it has, since a member
+requiring a module the build lacks fails every requirer of the group, and a manifest cannot
+record what a build has - its key does not see the build's artifacts. The text collector
+(`getAllRequireReq`) and the parser (`ast_requireModuleGroup`) expand the same list into one
+require per member, the group's guard and `public` on each, a member's own guard on that member;
+a member resolves and fails as a require spelled by hand would, and a group nothing registered
+adds nothing. The collector reads `require[group]` with no space as the parser does. The
+requirer calls the members through `daslib/module_group`, whose `call_module_group` reads the
+same list at macro time (`module_group_for_each_member`, the `rtti` module, a path guard
+resolved through the compiling program's own access) and emits one qualified call per member, so
+the expansion and the calls agree on the set. Membership is tree-level, so the answer does not
+depend on the walk order - a module joins from its own descriptor, and the requirer names only
+the group; the module-cache record stamps the expansion (sec.1), so a member joining later
+re-parses the modules that require the group. With
`DAS_TRACE_MODULE_LOAD=1` the scan prints one line per descriptor - `replayed N row(s) in
(shared module load , deferred K)`, `compiled (), manifest written (N row(s))`,
`compiled (no_manifest)`, `compiled (manifests ignored)`, or why a manifest was not written -
@@ -131,3 +168,84 @@ and a deferred load prints `[module] require : loading the deferred annotations = cloneAnnotationList(annotations);
cfun->result = new TypeDecl(*result);
cfun->body = body->clone();
- cfun->index = -1;
cfun->totalStackSize = 0;
cfun->totalGenLabel = totalGenLabel;
cfun->at = at;
diff --git a/src/ast/ast_allocate_stack.cpp b/src/ast/ast_allocate_stack.cpp
index ba6e151aad..4f6c934dee 100644
--- a/src/ast/ast_allocate_stack.cpp
+++ b/src/ast/ast_allocate_stack.cpp
@@ -138,8 +138,8 @@ namespace das {
virtual bool canVisitStructureFieldInit ( Structure * ) override { return false; }
virtual bool canVisitArgumentInit ( Function * , const VariablePtr &, Expression * ) override { return false; }
virtual bool canVisitQuoteSubexpression ( ExprQuote * ) override { return false; }
- virtual bool canVisitGlobalVariable ( Variable * var ) override { return isEverything || var->used; }
- virtual bool canVisitFunction ( Function * fun ) override { return !fun->isTemplate && !fun->stub && (isEverything || fun->used); }
+ virtual bool canVisitGlobalVariable ( Variable * var ) override { return isEverything || program->isUsed(var); }
+ virtual bool canVisitFunction ( Function * fun ) override { return !fun->isTemplate && !fun->stub && (isEverything || program->isUsed(fun)); }
// function
virtual void preVisit ( Function * f ) override {
Visitor::preVisit(f);
@@ -279,7 +279,7 @@ namespace das {
virtual bool canVisitQuoteSubexpression ( ExprQuote * ) override { return false; }
virtual bool canVisitGlobalVariable ( Variable * var ) override {
if ( var->stackResolved ) return false;
- if ( !var->used && !isEverything ) return false;
+ if ( !program->isUsed(var) && !isEverything ) return false;
var->stackResolved = isPermanent;
return true;
}
@@ -287,7 +287,7 @@ namespace das {
if ( fun->stub ) return false;
if ( fun->isTemplate ) return false;
if ( fun->stackResolved ) return false;
- if ( !fun->used && !isEverything ) return false;
+ if ( !program->isUsed(fun) && !isEverything ) return false;
fun->stackResolved = isPermanent;
return true;
}
@@ -379,7 +379,7 @@ namespace das {
stackTop = sizeof(Prologue);
pushSp();
if ( log ) {
- if (!func->used) logs << "unused ";
+ if (!program->isUsed(func)) logs << "unused ";
logs << func->describe() << "\n";
}
}
@@ -1036,15 +1036,17 @@ namespace das {
class AllocateConstString : public Visitor {
public:
+ AllocateConstString ( Program * prog ) : program(prog) {}
+ Program * program = nullptr;
uint32_t bytesTotal = 0;
das_hash_set uniStr;
public:
virtual bool canVisitStructureFieldInit ( Structure * ) override { return false; }
virtual bool canVisitArgumentInit ( Function * , const VariablePtr &, Expression * ) override { return false; }
virtual bool canVisitQuoteSubexpression ( ExprQuote * ) override { return false; }
- virtual bool canVisitGlobalVariable ( Variable * var ) override { return var->used; }
+ virtual bool canVisitGlobalVariable ( Variable * var ) override { return program->isUsed(var); }
virtual bool canVisitFunction ( Function * fun ) override {
- return !fun->isTemplate && fun->used;
+ return !fun->isTemplate && program->isUsed(fun);
}
void allocateString ( const string & message ) {
if ( !message.empty() ) {
@@ -1125,9 +1127,10 @@ namespace das {
class MarkTempStrings : public Visitor {
public:
- MarkTempStrings ( Function * wrapperFn, bool insertWrappers_, bool everything_ )
- : wrapper(wrapperFn), insertWrappers(insertWrappers_), isEverything(everything_) {}
+ MarkTempStrings ( Program * prog, Function * wrapperFn, bool insertWrappers_, bool everything_ )
+ : program(prog), wrapper(wrapperFn), insertWrappers(insertWrappers_), isEverything(everything_) {}
protected:
+ Program * program = nullptr;
Function * wrapper = nullptr;
bool insertWrappers = false;
bool isEverything = false;
@@ -1137,9 +1140,9 @@ namespace das {
virtual bool canVisitStructureFieldInit ( Structure * ) override { return false; }
virtual bool canVisitArgumentInit ( Function * , const VariablePtr &, Expression * ) override { return false; }
virtual bool canVisitQuoteSubexpression ( ExprQuote * ) override { return false; }
- virtual bool canVisitGlobalVariable ( Variable * var ) override { return isEverything || var->used; }
+ virtual bool canVisitGlobalVariable ( Variable * var ) override { return isEverything || program->isUsed(var); }
virtual bool canVisitFunction ( Function * fun ) override {
- return !fun->isTemplate && !fun->stub && (isEverything || fun->used);
+ return !fun->isTemplate && !fun->stub && (isEverything || program->isUsed(fun));
}
das_hash_map siteOf;
bool isWrapperCall ( Expression * e ) const {
@@ -1282,18 +1285,19 @@ namespace das {
// (each nested temp dies before the next link queues).
class WrapLetTempStrings : public Visitor {
public:
- WrapLetTempStrings ( Function * wrapperFn, bool everything_ )
- : wrapper(wrapperFn), isEverything(everything_) {}
+ WrapLetTempStrings ( Program * prog, Function * wrapperFn, bool everything_ )
+ : program(prog), wrapper(wrapperFn), isEverything(everything_) {}
protected:
+ Program * program = nullptr;
Function * wrapper = nullptr;
bool isEverything = false;
// same gates as MarkTempStrings above - this pass mutates let initializers
virtual bool canVisitStructureFieldInit ( Structure * ) override { return false; }
virtual bool canVisitArgumentInit ( Function * , const VariablePtr &, Expression * ) override { return false; }
virtual bool canVisitQuoteSubexpression ( ExprQuote * ) override { return false; }
- virtual bool canVisitGlobalVariable ( Variable * var ) override { return isEverything || var->used; }
+ virtual bool canVisitGlobalVariable ( Variable * var ) override { return isEverything || program->isUsed(var); }
virtual bool canVisitFunction ( Function * fun ) override {
- return !fun->isTemplate && !fun->stub && (isEverything || fun->used);
+ return !fun->isTemplate && !fun->stub && (isEverything || program->isUsed(fun));
}
virtual ExpressionPtr visit ( ExprBlock * block ) override {
auto & stmts = block->list;
@@ -1358,23 +1362,23 @@ namespace das {
if ( auto bmod = Module::require("$") ) {
wrapperFn = bmod->findUniqueFunction("_temp_string_result");
}
- MarkTempStrings mts(wrapperFn, wrapperFn!=nullptr, everything);
+ MarkTempStrings mts(this, wrapperFn, wrapperFn!=nullptr, everything);
visit(mts);
if ( wrapperFn ) {
- WrapLetTempStrings wlt(wrapperFn, everything);
+ WrapLetTempStrings wlt(this, wrapperFn, everything);
visit(wlt);
}
}
// string heap
- AllocateConstString vstr;
+ AllocateConstString vstr(this);
for (auto & pm : library.modules) {
for ( auto & var : pm->globals.each() ) {
- if (var->used && var->init) {
+ if (isUsed(var) && var->init) {
var->init->visit(vstr);
}
}
for ( auto & func : pm->functions.each() ) {
- if (func->used) {
+ if (isUsed(func)) {
func->visit(vstr);
}
}
@@ -1389,12 +1393,14 @@ namespace das {
// adjust stack size for all the used variables
for (auto & pm : library.modules) {
for ( auto & var : pm->globals.each() ) {
- if ( var->used ) {
+ if ( isUsed(var) ) {
globalInitStackSize = das::max(globalInitStackSize, var->initStackSize);
}
}
}
- // allocate used variables and functions indices
+ // allocate used variables and functions indices - per pass, so -1 means this pass never saw it
+ functionIndices.clear();
+ variableIndices.clear();
totalVariables = 0;
totalFunctions = 0;
auto log = options.getBoolOption("log_stack");
@@ -1404,10 +1410,11 @@ namespace das {
}
for (auto & pm : library.modules) {
for ( auto & func : pm->functions.each() ) {
- if ( func->used && !func->builtIn && !func->isTemplate ) {
- func->index = totalFunctions++;
+ if ( isUsed(func) && !func->builtIn && !func->isTemplate ) {
+ auto index = totalFunctions++;
+ setIndex(func, index);
if ( log ) {
- logs << "\t" << func->index << "\t" << func->totalStackSize << "\t" << func->getMangledName();
+ logs << "\t" << index << "\t" << func->totalStackSize << "\t" << func->getMangledName();
if ( func->init ) {
logs << " [init";
if (func->lateInit ) logs << "(late)";
@@ -1423,7 +1430,7 @@ namespace das {
}
}
else {
- func->index = -2;
+ setIndex(func, -2);
}
}
}
@@ -1432,15 +1439,16 @@ namespace das {
}
library.foreach_in_order([&](Module * pm){
for ( auto & var : pm->globals.each() ) {
- if (var->used) {
- var->index = totalVariables++;
+ if (isUsed(var)) {
+ auto index = totalVariables++;
+ setIndex(var, index);
if ( log ) {
- logs << "\t" << var->index << "\t" << var->stackTop << "\t"
+ logs << "\t" << index << "\t" << var->stackTop << "\t"
<< var->type->getSizeOf() << "\t" << var->getMangledName() << "\n";
}
}
else {
- var->index = -2;
+ setIndex(var, -2);
}
}
return true;
diff --git a/src/ast/ast_const_folding.cpp b/src/ast/ast_const_folding.cpp
index 1317c694f9..63df4cc5f0 100644
--- a/src/ast/ast_const_folding.cpp
+++ b/src/ast/ast_const_folding.cpp
@@ -1675,7 +1675,7 @@ namespace das {
return Visitor::visit(expr);
}
}
- if ( expr->func->index==-1 ) {
+ if ( runProgram->indexOf(expr->func)==-1 ) {
runProgram->error("internal compilation error, folding symbol was not marked as used","","",
expr->at, CompilationError::internal_function);
return Visitor::visit(expr);
diff --git a/src/ast/ast_derive_alias.cpp b/src/ast/ast_derive_alias.cpp
index 06f1681d38..66d5fdc374 100644
--- a/src/ast/ast_derive_alias.cpp
+++ b/src/ast/ast_derive_alias.cpp
@@ -41,7 +41,7 @@ namespace das {
mod->functions.foreach([&](const FunctionPtr & gfunc){
// not built-in, used, address taken, can potentially alias, compatible
if ( gfunc->isTemplate ) return;
- if ( !gfunc->builtIn && gfunc->used && gfunc->addressTaken && !gfunc->aliasCMRES && isCompatibleFunction(gfunc, inv) ) {
+ if ( !gfunc->builtIn && gfunc->addressTaken && !gfunc->aliasCMRES && program->isUsed(gfunc) && isCompatibleFunction(gfunc, inv) ) {
appendIndVariables(gfunc, sources, accessed);
}
});
@@ -69,7 +69,7 @@ namespace das {
mod->functions.foreach([&](const FunctionPtr & gfunc){
// not built-in, used, address taken, can potentially alias, compatible
if ( gfunc->isTemplate ) return;
- if ( !gfunc->builtIn && gfunc->used && gfunc->addressTaken && !gfunc->aliasCMRES && gfunc->lambda && isCompatibleLambdaFunction(gfunc, inv) ) {
+ if ( !gfunc->builtIn && gfunc->addressTaken && !gfunc->aliasCMRES && gfunc->lambda && program->isUsed(gfunc) && isCompatibleLambdaFunction(gfunc, inv) ) {
appendIndVariables(gfunc, sources, accessed);
}
});
@@ -317,7 +317,7 @@ namespace das {
virtual bool canVisitQuoteSubexpression ( ExprQuote * ) override { return false; }
virtual bool canVisitGlobalVariable ( Variable * var ) override {
if ( var->aliasesResolved ) return false;
- if ( !var->used && !isEverything ) return false;
+ if ( !program->isUsed(var) && !isEverything ) return false;
var->aliasesResolved = isPermanent;
return true;
}
@@ -328,7 +328,7 @@ namespace das {
if ( fun->stub ) return false;
if ( fun->isTemplate ) return false;
if ( fun->aliasesResolved ) return false;
- if ( !fun->used && !isEverything ) return false;
+ if ( !program->isUsed(fun) && !isEverything ) return false;
fun->aliasesResolved = isPermanent;
return true;
}
@@ -623,17 +623,11 @@ namespace das {
library.foreach([&](Module * mod){
if ( logAliasing ) logs << "module " << mod->name << ":\n";
mod->functions.foreach([&](const FunctionPtr & func){
- if ( func->builtIn || !func->used || func->isTemplate ) return;
+ if ( func->builtIn || !isUsed(func) || func->isTemplate ) return;
deriveAliasing(func, logs, logAliasing);
});
return true;
},"*");
- /*
- thisModule->functions.foreach([&](const FunctionPtr & func) {
- if ( func->builtIn || !func->used ) return;
- deriveAliasing(func, logs, logAliasing);
- });
- */
AliasMarker marker(this, permanent, everything, logs);
visit(marker);
if ( logAliasing ) logs << "\n";
diff --git a/src/ast/ast_export.cpp b/src/ast/ast_export.cpp
index 4fefc5f3d4..27022f2a71 100644
--- a/src/ast/ast_export.cpp
+++ b/src/ast/ast_export.cpp
@@ -7,7 +7,7 @@ namespace das {
class ClearUnusedSymbols : public Visitor {
public:
- ClearUnusedSymbols ( Module * tm ) : thisModule(tm) {}
+ ClearUnusedSymbols ( Program * prog, Module * tm ) : program(prog), thisModule(tm) {}
virtual bool canVisitFunction ( Function * fun ) override {
return !fun->stub && !fun->isTemplate; // we don't do a thing with templates
}
@@ -22,7 +22,7 @@ namespace das {
virtual bool canVisitArgumentInit ( Function *, const VariablePtr &, Expression * ) override { return true; }
virtual void preVisit(ExprAddr * expr) override {
Visitor::preVisit(expr);
- if ( expr->func && !expr->func->used && !expr->func->builtIn ) {
+ if ( expr->func && !program->isUsed(expr->func) && !expr->func->builtIn ) {
expr->func = nullptr;
}
}
@@ -33,7 +33,7 @@ namespace das {
// SIGSEGV in getMangledName under --ser)
virtual void preVisit(ExprMakeStruct * expr) override {
Visitor::preVisit(expr);
- if ( expr->constructor && !expr->constructor->used && !expr->constructor->builtIn ) {
+ if ( expr->constructor && !program->isUsed(expr->constructor) && !expr->constructor->builtIn ) {
expr->constructor = nullptr;
}
}
@@ -43,7 +43,7 @@ namespace das {
// like math::PI - stay untouched and keep their valid metadata
virtual void preVisit(ExprVar * expr) override {
Visitor::preVisit(expr);
- if ( expr->variable && expr->variable->module == thisModule && !expr->variable->used ) {
+ if ( expr->variable && expr->variable->module == thisModule && !program->isUsed(expr->variable) ) {
expr->variable = nullptr;
}
}
@@ -51,18 +51,19 @@ namespace das {
Visitor::preVisitExpression(expr);
if ( expr->rtti_isCallFunc() ) {
auto cfe = static_cast(expr);
- if ( cfe->func && !cfe->func->used && !cfe->func->builtIn ) {
+ if ( cfe->func && !program->isUsed(cfe->func) && !cfe->func->builtIn ) {
cfe->func = nullptr;
}
}
}
protected:
+ Program * program = nullptr;
Module * thisModule = nullptr;
};
class MarkSymbolUse : public Visitor {
public:
- MarkSymbolUse ( bool bid ) : builtInDependencies(bid) {
+ MarkSymbolUse ( Program * prog, bool bid ) : program(prog), builtInDependencies(bid) {
}
__forceinline void push ( Variable* pvar ) {
if ( !tw ) return;
@@ -80,9 +81,9 @@ namespace das {
}
void propageteVarUse(Variable * var) {
DAS_ASSERT(var);
- if (var->used) return;
+ if (program->isUsed(var)) return;
push(var);
- var->used = true;
+ program->setUsed(var, true);
for (const auto & gv : var->useGlobalVariables) {
propageteVarUse(gv);
}
@@ -94,10 +95,10 @@ namespace das {
void propagateFunctionUse(Function * fn) {
DAS_ASSERT(fn);
if (fn->isTemplate) return;
- if (fn->used) return;
+ if (program->isUsed(fn)) return;
if (fn->builtIn) return;
push(fn);
- fn->used = true;
+ program->setUsed(fn, true);
for (const auto & gv : fn->useGlobalVariables) {
propageteVarUse(gv);
}
@@ -112,8 +113,8 @@ namespace das {
void markVarsUsed( ModuleLibrary & lib, bool forceAll ){
lib.foreach([&](Module * pm) {
for ( auto & var : pm->globals.each() ) {
- if ( forceAll || var->used || isVarExported(var) ) {
- var->used = false;
+ if ( forceAll || program->isUsed(var) || isVarExported(var) ) {
+ program->setUsed(var, false);
propageteVarUse(var);
}
}
@@ -133,7 +134,7 @@ namespace das {
}
void markModuleVarsUsed( ModuleLibrary &, Module * inWhichModule ) {
for ( auto & var : inWhichModule->globals.each() ) {
- var->used = false;
+ program->setUsed(var, false);
propageteVarUse(var);
}
}
@@ -152,14 +153,14 @@ namespace das {
mod.functionsByName.clear();
// mod.globals.clear();
for ( auto & fn : functions.each() ) {
- if ( fn->used ) {
+ if ( program->isUsed(fn) ) {
if ( !mod.addFunction(fn, true) ) {
program->error("internal error, failed to add function " + fn->name,"","", fn->at, CompilationError::internal_function );
}
}
}
for ( auto & var : globals.each() ) {
- if ( var->used ) {
+ if ( program->isUsed(var) ) {
if ( !mod.addVariable(var, true) ) {
program->error("internal error, failed to add variable " + var->name,"","", var->at, CompilationError::internal_variable );
}
@@ -189,7 +190,7 @@ namespace das {
gVar = var;
var->useFunctions.clear();
var->useGlobalVariables.clear();
- var->used = false;
+ program->setUsed(var, false);
}
virtual VariablePtr visitGlobalLet(const VariablePtr & var) override {
gVar = nullptr;
@@ -201,7 +202,7 @@ namespace das {
func = f;
func->useFunctions.clear();
func->useGlobalVariables.clear();
- func->used = false;
+ program->setUsed(f, false);
func->callCaptureString = false;
func->hasStringBuilder = false;
DAS_ASSERTF(!func->builtIn, "visitor should never call 'visit' on builtin function at top level.");
@@ -316,20 +317,14 @@ namespace das {
};
void Program::clearSymbolUse() {
- for (auto & pm : library.modules) {
- for ( auto & var : pm->globals.each() ) {
- var->used = false;
- }
- for ( auto & fn : pm->functions.each() ) {
- fn->used = false;
- }
- }
+ usedFunctions.clear();
+ usedVariables.clear();
}
void Program::markModuleSymbolUse(TextWriter * logs) {
// this module public, this module export, this module init\shutdown
clearSymbolUse();
- MarkSymbolUse vis(false);
+ MarkSymbolUse vis(this, false);
vis.tw = logs;
visit(vis);
vis.markModuleUsedFunctions(library, thisModule.get());
@@ -339,7 +334,7 @@ namespace das {
void Program::markMacroSymbolUse(TextWriter * logs) {
// this module macro init
clearSymbolUse();
- MarkSymbolUse vis(false);
+ MarkSymbolUse vis(this, false);
vis.tw = logs;
visit(vis);
vis.markUsedFunctions(library, false, true, thisModule.get());
@@ -348,7 +343,7 @@ namespace das {
void Program::markExecutableSymbolUse(TextWriter * logs) {
clearSymbolUse();
- MarkSymbolUse vis(false);
+ MarkSymbolUse vis(this, false);
vis.tw = logs;
visit(vis);
vis.markUsedFunctions(library, false, false, nullptr);
@@ -357,7 +352,7 @@ namespace das {
void Program::markFoldingSymbolUse(const vector & needRun, TextWriter * logs) {
clearSymbolUse();
- MarkSymbolUse vis(false);
+ MarkSymbolUse vis(this, false);
vis.tw = logs;
visit(vis);
for ( auto fun : needRun ) {
@@ -367,7 +362,7 @@ namespace das {
void Program::markSymbolUse(bool builtInSym, bool forceAll, bool initThis, Module * macroModule, TextWriter * logs) {
clearSymbolUse();
- MarkSymbolUse vis(builtInSym);
+ MarkSymbolUse vis(this, builtInSym);
vis.tw = logs;
visit(vis);
vis.markUsedFunctions(library, forceAll, initThis, macroModule);
@@ -376,9 +371,9 @@ namespace das {
void Program::removeUnusedSymbols() {
if ( options.getBoolOption("remove_unused_symbols",true) ) {
- ClearUnusedSymbols cvis(thisModule.get());
+ ClearUnusedSymbols cvis(this, thisModule.get());
visit(cvis);
- MarkSymbolUse vis(false);
+ MarkSymbolUse vis(this, false);
vis.RemoveUnusedSymbols(*thisModule);
}
}
@@ -387,12 +382,12 @@ namespace das {
logs << "USED SYMBOLS ARE:\n";
for (auto & pm : library.modules) {
for ( auto & var : pm->globals.each() ) {
- if ( var->used ) {
+ if ( isUsed(var) ) {
logs << "let " << var->module->name << "::" << var->name << ": " << var->type->describe() << "\n";
}
}
for ( auto & func : pm->functions.each() ) {
- if ( func->used ) {
+ if ( isUsed(func) ) {
logs << func->module->name << "::" << func->describe() << "\n";
}
}
diff --git a/src/ast/ast_lint.cpp b/src/ast/ast_lint.cpp
index 56eee2cbff..6adf6364d3 100644
--- a/src/ast/ast_lint.cpp
+++ b/src/ast/ast_lint.cpp
@@ -613,6 +613,7 @@ namespace das {
}
}
}
+ das_hash_set initializedGlobals;
virtual void preVisitGlobalLetInit ( const VariablePtr & var, Expression * that ) override {
Visitor::preVisitGlobalLetInit(var,that);
globalVar = var;
@@ -640,16 +641,16 @@ namespace das {
program->error("[init] is disabled in the options or CodeOfPolicies", "", "",
var->at, CompilationError::cant_global);
}
- globalVar->index = -3; // initialized. -1 by default
+ initializedGlobals.insert(&*globalVar);
globalVar = nullptr;
return Visitor::visitGlobalLetInit(var,that);;
}
virtual void preVisit(ExprVar * expr) override {
Visitor::preVisit(expr);
if ( globalVar && expr->isGlobalVariable() ) {
- if ( expr->variable->index!=-3 ) {
+ if ( initializedGlobals.find(&*expr->variable)==initializedGlobals.end() ) {
if ( expr->variable->module==globalVar->module ) {
- program->error("global variable " + expr->name + " is initialized after " + globalVar->name + " (" + to_string(expr->variable->index) + ")",
+ program->error("global variable " + expr->name + " is initialized after " + globalVar->name,
"", "", expr->at, CompilationError::invalid_global);
}
}
diff --git a/src/ast/ast_module.cpp b/src/ast/ast_module.cpp
index 38b14624b7..8b62e4f37e 100644
--- a/src/ast/ast_module.cpp
+++ b/src/ast/ast_module.cpp
@@ -8,8 +8,10 @@
#include "daScript/misc/handle_registry.h"
#include "daScript/simulate/simulate_fusion.h"
#include "daScript/ast/dyn_modules.h"
+#include "daScript/ast/ast_serializer.h"
#include
+#include
namespace das {
@@ -193,6 +195,30 @@ namespace das {
return g_deferredModuleLoader && g_deferredModuleLoader(name) && Module::requireEx(name, false);
}
+ // ARCHITECTURE.md sec.2 - process-wide, like the native paths: a descriptor registers once per process
+ static das_map> g_moduleGroups;
+ static mutex g_moduleGroupsMutex;
+
+ void registerModuleGroupMember ( const string & group, const string & member, const string & guard ) {
+ lock_guard guard_(g_moduleGroupsMutex);
+ auto & members = g_moduleGroups[group];
+ for ( auto & m : members ) {
+ if ( m.member == member ) return;
+ }
+ members.push_back({member, guard});
+ }
+
+ vector getModuleGroupMembers ( const string & group ) {
+ lock_guard guard(g_moduleGroupsMutex);
+ auto it = g_moduleGroups.find(group);
+ if ( it == g_moduleGroups.end() ) return vector();
+ vector members = it->second;
+ sort(members.begin(), members.end(), [](const ModuleGroupMember & a, const ModuleGroupMember & b){
+ return a.member < b.member;
+ });
+ return members;
+ }
+
void Module::Initialize() {
daScriptEnvironment::ensure();
static bool atexit_registered = (atexit(daslang_atexit_audit), true);
diff --git a/src/ast/ast_parse.cpp b/src/ast/ast_parse.cpp
index 331e1df35f..1a4cbd45a4 100644
--- a/src/ast/ast_parse.cpp
+++ b/src/ast/ast_parse.cpp
@@ -219,7 +219,7 @@ namespace das {
bool isMod = !isReq && !isInc && (memcmp(src, "module", 6)==0);
if ( isReq || isInc ) {
src += 7;
- if ( isspace(src[0]) ) {
+ if ( isspace(src[0]) || (isReq && src[0]=='[') ) {
while ( src < src_end && isspace(src[0]) ) {
src ++;
}
@@ -247,6 +247,47 @@ namespace das {
continue;
}
}
+ // ARCHITECTURE.md sec.2
+ auto guardAvailable = [&](const string & guard) {
+ if ( guard.empty() ) return true;
+ if ( guard.find('/')!=string::npos ) {
+ auto ginfo = access->getModuleInfo(guard, fi->name);
+ return !ginfo.fileName.empty() && access->getFileInfo(ginfo.fileName) != nullptr;
+ }
+ return guardModuleAvailable(guard);
+ };
+ auto guardTaken = [&]() {
+ return !hasReqGuard || guardAvailable(reqGuard);
+ };
+ auto publicFollows = [&]() {
+ while ( src < src_end && isspace(src[0]) ) { // the parser reads tokens: any whitespace before `public`
+ src ++;
+ }
+ return src + 6 < src_end && memcmp(src, "public", 6) == 0;
+ };
+ if ( isReq && src[0]=='[' ) {
+ // ARCHITECTURE.md sec.2
+ src ++;
+ while ( src < src_end && isspace(src[0]) ) { // the parser reads tokens, so a newline inside the brackets is nothing
+ src ++;
+ }
+ string group;
+ while ( src < src_end && (isalnumE(src[0]) || src[0]=='_') ) {
+ group += *src ++;
+ }
+ while ( src < src_end && isspace(src[0]) ) {
+ src ++;
+ }
+ if ( src < src_end && src[0]==']' && !group.empty() && guardTaken() ) {
+ src ++;
+ bool isPublic = publicFollows();
+ for ( const auto & member : getModuleGroupMembers(group) ) {
+ if ( !guardAvailable(member.guard) ) continue;
+ req.push_back({member.member, line, chain, isPublic});
+ }
+ }
+ continue;
+ }
if ( src[0]=='_' || isalphaE(src[0]) || src[0] == '%' || src[0] == '.' || src[0]=='/' ) {
string mod;
mod += *src++;
@@ -254,28 +295,10 @@ namespace das {
mod += *src ++;
}
if ( isReq ) {
- // guarded optional require. Path guard (contains '/'): proceed only when
- // the guard's OWN file resolves — the rail for pure-das packages (nothing
- // C++ to guard on) and cross-package dependency witnesses. Plain-name
- // guard: proceed only when the build has the module (guardModuleAvailable,
- // src/ast/ARCHITECTURE.md sec.2); no target-resolvability fallback (module
- // source dirs exist in every checkout regardless of build config).
- // Otherwise skip silently. Must match ast_requireModule (parser_impl.cpp).
- if ( hasReqGuard && reqGuard.find('/')!=string::npos ) {
- auto ginfo = access->getModuleInfo(reqGuard, fi->name);
- if ( ginfo.fileName.empty() || !access->getFileInfo(ginfo.fileName) ) {
- continue;
- }
- } else if ( hasReqGuard && !guardModuleAvailable(reqGuard) ) {
+ if ( !guardTaken() ) {
continue;
}
- bool isPublic = false;
- while ( src < src_end && src[0] == ' ' ) {
- src ++;
- }
- if ( src + 6 < src_end && memcmp(src, "public", 6) == 0 ) {
- isPublic = true;
- }
+ bool isPublic = publicFollows();
req.push_back({mod, line, chain, isPublic});
} else if ( isInc ) {
string incFileName = access->getIncludeFileName(fi->name,mod);
@@ -676,6 +699,18 @@ namespace das {
}
}
+ // ARCHITECTURE.md sec.1
+ static vector collectRequireNames ( FileInfo * fi, const FileAccessPtr & access ) {
+ vector names;
+ if ( fi ) {
+ string modName;
+ vector chain;
+ for ( auto & rec : getAllRequire(fi, modName, chain, access) ) names.push_back(rec.name);
+ }
+ sort(names.begin(), names.end());
+ return names;
+ }
+
bool trySerializeProgramModule (
ProgramPtr & program,
const FileAccessPtr & access,
@@ -722,6 +757,8 @@ namespace das {
uint64_t payload_size = 0;
uint32_t depCount = 0;
vector> savedDeps;
+ uint32_t reqCount = 0;
+ vector savedReq;
if ( !serializer_read->trySerialize([&](AstSerializer & serializer) {
serializer << saved_hash;
serializer << saved_size;
@@ -737,10 +774,17 @@ namespace das {
serializer << get<2>(dep);
}
}
+ serializer << reqCount;
+ if ( depCount <= SER_MAX_MACRO_DEPS && reqCount <= SER_MAX_MACRO_DEPS ) {
+ savedReq.resize(reqCount);
+ for ( auto & req : savedReq ) {
+ serializer << req;
+ }
+ }
serializer << payload_size;
- }) || depCount > SER_MAX_MACRO_DEPS ) {
+ }) || depCount > SER_MAX_MACRO_DEPS || reqCount > SER_MAX_MACRO_DEPS ) {
serializer_read->seenNewModule = true;
- serializer_read->failed = depCount > SER_MAX_MACRO_DEPS;
+ serializer_read->failed = depCount > SER_MAX_MACRO_DEPS || reqCount > SER_MAX_MACRO_DEPS;
if ( !serializer_read->quietCache ) logs << "ser: read failed '" << fileName << "'\n";
return false;
}
@@ -779,17 +823,34 @@ namespace das {
}
return false;
}
+ // ARCHITECTURE.md sec.1
+ {
+ auto currentReq = collectRequireNames(access->getFileInfo(fileName), access);
+ sort(savedReq.begin(), savedReq.end());
+ if ( currentReq != savedReq ) {
+ serializer_read->seenNewModule = true;
+ serializer_read->failed = true;
+ serializer_read->cutoffFile = fileName;
+ serializer_read->cutoffReason = "require set changed";
+ if ( !serializer_read->quietCache ) {
+ logs << "ser: require set changed '" << fileName << "': " << savedReq.size() << " recorded, " << currentReq.size() << " now\n";
+ }
+ return false;
+ }
+ }
size_t payload_start = serializer_read->buffer->bufferPos;
// no per-module logging on the happy path - the standalone sink has no level
// threshold, so a line here prints once per warm module; the verdict line and the
// failure branches below carry everything a human needs
deriveSourcePolicies(access, fileName, program->policies);
+ serializer_read->readingRecord ++;
bool read_ok = serializer_read->trySerialize([&](AstSerializer & serializer) {
serializer.thisModuleGroup = &libGroup;
serializer.fileAccess = access.get();
serializer.serializeProgram(program, libGroup);
});
+ serializer_read->readingRecord --;
if ( read_ok && !program->failed() && !serializer_read->failed ) {
serializer_read->servedModules ++;
@@ -798,7 +859,7 @@ namespace das {
// must round-trip through the deserialized program or the next write drops them
program->moduleCacheDependencies = das::move(savedDeps);
if ( serializer_write != nullptr ) {
- serializer_write->parsedModules.push_back({fileName, file_hash, file_size, program, program->thisModule.get()});
+ serializer_write->parsedModules.push_back({fileName, file_hash, file_size, program, program->thisModule.get(), das::move(savedReq)});
}
return true;
}
@@ -1087,6 +1148,7 @@ namespace das {
auto & serializer_read = daScriptEnvironment::getBound()->serializer_read;
uint64_t macroSim0 = serializer_read ? serializer_read->totMacroTime : 0;
if ( trySerializeProgramModule(program, access, fileName, libGroup, logs) ) {
+ program->access = access; // the read replaced the program object
auto readT = get_time_usec(time0);
auto macroSimT = int64_t(serializer_read->totMacroTime - macroSim0);
*totCacheRead += readT;
@@ -1104,6 +1166,7 @@ namespace das {
}
int err;
+ program->access = access; // a failed read replaced the program object above
daScriptEnvironment::getBound()->g_Program = program;
daScriptEnvironment::getBound()->g_compilerLog = &logs;
daScriptEnvironment::getBound()->g_compilingFileName = fileName.c_str();
@@ -1395,8 +1458,9 @@ namespace das {
logs << "compiler took " << dt << ", " << fileName << "\n";
}
auto & serializer_write = daScriptEnvironment::getBound()->serializer_write;
- if ( serializer_write != nullptr ) {
- serializer_write->parsedModules.push_back({fileName, file_hash, file_size, program, program->thisModule.get()});
+ if ( serializer_write != nullptr && !program->failed() ) { // a failed module is no record (a persistent stream's later writeback would carry it)
+ serializer_write->parsedModules.push_back({fileName, file_hash, file_size, program, program->thisModule.get(),
+ collectRequireNames(access->getFileInfo(fileName), access)});
}
return program;
}
@@ -1530,8 +1594,10 @@ namespace das {
*serializer_write << magic;
*serializer_write << version;
}
- for ( auto & parsedModule : serializer_write->parsedModules ) {
- auto & [fileName, fileHash, fileSize, program, thisModule] = parsedModule;
+ // from the cursor: a late walk (requireModuleNow) writes back mid-compile, and the host's
+ // writeback at its end must not repeat those records
+ for ( size_t pi = serializer_write->writtenModules; pi < serializer_write->parsedModules.size(); ++pi ) {
+ auto & [fileName, fileHash, fileSize, program, thisModule, requireNames] = serializer_write->parsedModules[pi];
*serializer_write << fileHash;
*serializer_write << fileSize;
*serializer_write << const_cast(fileName);
@@ -1542,6 +1608,12 @@ namespace das {
*serializer_write << get<1>(dep);
*serializer_write << get<2>(dep);
}
+ // ARCHITECTURE.md sec.1
+ uint32_t reqCount = uint32_t(requireNames.size());
+ *serializer_write << reqCount;
+ for ( auto & req : requireNames ) {
+ *serializer_write << req;
+ }
// record length, backpatched after the payload: lets the reader skip a record
// that fails to deserialize for an UNCHANGED file and keep serving later ones.
// Fixed-width u64 on purpose - an adaptive size could not be patched in place.
@@ -1561,6 +1633,7 @@ namespace das {
payload_size = uint64_t(serializer_write->buffer->writingSize() - payload_start);
serializer_write->buffer->patch(len_at, &payload_size, sizeof(payload_size));
}
+ serializer_write->writtenModules = serializer_write->parsedModules.size();
}
void addRttiRequireVariable ( ProgramPtr res, string fileName, const LineInfo & at ) {
@@ -1575,9 +1648,9 @@ namespace das {
rtti_require->type = new TypeDecl(Type::tString, at);
rtti_require->init = new ExprConstString(at, ss.str());
rtti_require->init->type = new TypeDecl(Type::tString, at);
- rtti_require->used = true;
rtti_require->private_variable = true;
res->thisModule->addVariable(rtti_require);
+ res->setUsed(rtti_require, true);
}
void reportChain ( TextWriter & tw, const vector & chain ) {
@@ -1807,6 +1880,150 @@ namespace das {
return parseDaScriptNoInfer(fileName, modName, access, logs, libGroup, policies.export_all, false, policies);
}
+ static recursive_mutex g_requireModuleNowMutex;
+
+ static void logProgramErrors ( const ProgramPtr & program, TextWriter & logs ) {
+ for ( auto & err : program->errors ) {
+ logs << err.at.describe() << ": " << err.what << "\n";
+ if ( !err.extra.empty() ) logs << err.extra << "\n";
+ }
+ }
+
+ // ARCHITECTURE.md sec.3 - what a nested compile rebinds on the environment, put back on every exit;
+ // a walk nested in a parse or a record read also hides the stream for its duration
+ struct LateRequireEnvScope {
+ daScriptEnvironment * env;
+ ProgramPtr program;
+ TextWriter * log;
+ const char * fileName;
+ const char * moduleName;
+ AstSerializer * read = nullptr;
+ AstSerializer * write = nullptr;
+ bool streamHidden = false;
+ LateRequireEnvScope ( daScriptEnvironment * e ) : env(e), program(e->g_Program), log(e->g_compilerLog),
+ fileName(e->g_compilingFileName), moduleName(e->g_compilingModuleName) {}
+ void hideStream () {
+ read = env->serializer_read;
+ write = env->serializer_write;
+ env->serializer_read = nullptr;
+ env->serializer_write = nullptr;
+ streamHidden = true;
+ }
+ ~LateRequireEnvScope () {
+ env->g_Program = program;
+ env->g_compilerLog = log;
+ env->g_compilingFileName = fileName;
+ env->g_compilingModuleName = moduleName;
+ if ( streamHidden ) {
+ env->serializer_read = read;
+ env->serializer_write = write;
+ }
+ }
+ };
+
+ // ARCHITECTURE.md sec.3
+ Module * requireModuleNow ( const string & requireName, const FileAccessPtr & access, TextWriter & logs, CodeOfPolicies policies ) {
+ lock_guard guard(g_requireModuleNowMutex);
+ verifyCodeOfPoliciesStamp(policies);
+ auto env = daScriptEnvironment::getBound();
+ // a relative target resolves against the file issuing the require, as a written require would
+ auto info = access->getModuleInfo(requireName, env->g_compilingFileName ? env->g_compilingFileName : "");
+ string modName = info.moduleName.empty() ? requireName : info.moduleName;
+ if ( auto mod = Module::requireEx(modName, true, requireName, info.fileName) ) return mod;
+ if ( info.fileName.empty() ) {
+ if ( auto loader = getDeferredModuleLoader(); loader && loader(modName) ) {
+ return Module::requireEx(modName, true, requireName, info.fileName);
+ }
+ logs << "require_module_now: module '" << requireName << "' not found\n";
+ return nullptr;
+ }
+ LateRequireEnvScope envScope(env);
+ // ARCHITECTURE.md sec.3 - a walk nested in a parse or a record read leaves the stream alone
+ bool nested = (env->g_Program && env->g_Program->isCompiling)
+ || (env->serializer_read && env->serializer_read->readingRecord > 0);
+ if ( nested ) envScope.hideStream();
+ Module * result = nullptr;
+ {
+ gc_guard compile_gc_scope;
+ GcCollectOnExit compile_gc_collect(compile_gc_scope);
+ ReuseCacheGuard rcg;
+ ModuleGroup libGroup;
+ vector req;
+ vector missing;
+ vector circular, notAllowed;
+ vector chain;
+ das_set dependencies;
+ das_hash_map namelessReq;
+ vector namelessMismatches;
+ string fileModName;
+ bool ok = addExtraDependency("builtin", get_builtin_path(), missing, circular, notAllowed, req,
+ dependencies, namelessReq, namelessMismatches, access, libGroup, policies, &logs);
+ ok = ok && getPrerequisits(info.fileName, access, fileModName, req, missing, circular, notAllowed, chain,
+ dependencies, namelessReq, namelessMismatches, libGroup, nullptr, 1, !policies.ignore_shared_modules);
+ if ( !ok ) {
+ logProgramErrors(reportPrerequisitesErrors(info.fileName, missing, circular, notAllowed, namelessMismatches, libGroup, policies), logs);
+ } else if ( !verifyModuleNamesUnique(req, logs) ) {
+ ok = false;
+ } else if ( !fileModName.empty() && fileModName != modName ) {
+ logs << "require_module_now: '" << info.fileName << "' declares module '" << fileModName
+ << "' - a module required after the walk is named by its file, so it declares `module " << modName << "`\n";
+ ok = false;
+ }
+ if ( ok ) {
+ disableSerializationOnDebugger(req);
+ for ( auto & mod : req ) {
+ if ( libGroup.findModule(mod.moduleName) ) continue;
+ auto program = parseDaScript(mod.fileName, mod.moduleName, access, logs, libGroup, true, true, policies);
+ policies.threadlock_context |= program->options.getBoolOption("threadlock_context", false);
+ if ( program->failed() ) {
+ logProgramErrors(program, logs);
+ ok = false;
+ break;
+ }
+ if ( program->thisModule->name.empty() ) {
+ program->library.renameModule(program->thisModule.get(), mod.moduleName);
+ program->thisModule->wasParsedNameless = true;
+ }
+ program->thisModule->fileName = mod.fileName;
+ program->thisModule->fromExtraDependency = mod.extraDepModule;
+ if ( program->promoteToBuiltin ) {
+ if ( !canShareModule(program) ) {
+ logProgramErrors(program, logs);
+ ok = false;
+ break;
+ }
+ program->thisModule->promoteToBuiltin(access, mod.requireName);
+ }
+ addNewModules(libGroup, program);
+ }
+ }
+ if ( ok ) {
+ auto program = parseDaScript(info.fileName, modName, access, logs, libGroup, true, true, policies);
+ if ( program->failed() ) {
+ logProgramErrors(program, logs);
+ } else if ( !program->promoteToBuiltin ) {
+ logs << "require_module_now: module '" << requireName << "' is not shared - a module required after the walk needs `module " << modName << " shared`\n";
+ } else if ( !canShareModule(program) ) {
+ logProgramErrors(program, logs);
+ } else {
+ if ( program->thisModule->name.empty() ) {
+ program->library.renameModule(program->thisModule.get(), modName);
+ program->thisModule->wasParsedNameless = true;
+ }
+ program->thisModule->fileName = info.fileName;
+ program->thisModule->promoteToBuiltin(access, requireName);
+ result = program->thisModule.get();
+ addNewModules(libGroup, program);
+ if ( env->serializer_write != nullptr
+ && (!env->serializer_read || env->serializer_read->failed || env->serializer_read->resumedCorrupt != 0) ) {
+ writebackModules(libGroup);
+ }
+ }
+ }
+ }
+ return result;
+ }
+
ProgramPtr compileDaScript ( const string & fileName,
const FileAccessPtr & access,
TextWriter & logs,
diff --git a/src/ast/ast_simulate.cpp b/src/ast/ast_simulate.cpp
index 01a6ad8b1d..6dd9c80bf7 100644
--- a/src/ast/ast_simulate.cpp
+++ b/src/ast/ast_simulate.cpp
@@ -27,6 +27,18 @@ namespace das
// fusion function pointers (defined here in main lib, set by fusion lib)
void (*g_fusionContextFn) ( Context & context, TextWriter & logs, bool enableFusion ) = nullptr;
void (*g_resetFusionEngineFn) () = nullptr;
+ // ARCHITECTURE.md sec.4
+ static __forceinline int32_t programIndexOf ( const Context & context, const Function * fn ) {
+ return context.thisProgram ? context.thisProgram->indexOf(fn) : -1;
+ }
+ static __forceinline int32_t programIndexOf ( const Context & context, const Variable * var ) {
+ return context.thisProgram ? context.thisProgram->indexOf(var) : -1;
+ }
+ // the callee's SimFunction: the program's slot, or - simulating outside any program - the context's table by hash
+ static __forceinline SimFunction * programFunction ( Context & context, Function * fn ) {
+ if ( auto index = programIndexOf(context, fn); index>=0 ) return context.getFunction(index);
+ return context.thisProgram ? nullptr : context.fnByMangledName(fn->getMangledNameHash());
+ }
// topological sort for the [init] nodes
struct InitSort {
@@ -817,8 +829,8 @@ namespace das
if ( mks->constructor ) {
uint32_t offset = mks->extraOffset + index*stride;
SimNode_CallBase * pCall = (SimNode_CallBase *) context.code->makeNodeUnrollAny(0, mks->at);
- DAS_ASSERT(mks->constructor->index!=-1 && "should have failed in type infer otherwise");
- pCall->fnPtr = context.getFunction(mks->constructor->index);
+ pCall->fnPtr = programFunction(context, mks->constructor);
+ DAS_ASSERT(pCall->fnPtr && "should have failed in type infer otherwise");
if ( mks->useCMRES ) {
pCall->cmresEval = context.code->makeNode(mks->at, offset);
} else if ( mks->useStackRef ) {
@@ -1208,7 +1220,7 @@ namespace das
context.thisProgram->error("internal compilation error, ExprAddr func is null", "", "", at, CompilationError::internal_function);
setE(expr, nullptr);
return expr;
- } else if ( expr->func->index<0 ) {
+ } else if ( programIndexOf(context,expr->func)<0 ) {
context.thisProgram->error("internal compilation error, ExprAddr func->index is unused", "", "", at, CompilationError::internal_function);
setE(expr, nullptr);
return expr;
@@ -1220,8 +1232,9 @@ namespace das
} temp;
temp.cval = v_zero();
if ( expr->func->module->isSolidContext ) {
- DAS_ASSERT(expr->func->index>=0 && "address of unsued function? how?");
- temp.mnh = expr->func->index;
+ auto index = programIndexOf(context,expr->func);
+ DAS_ASSERT(index>=0 && "address of unsued function? how?");
+ temp.mnh = index;
setE(expr, context.code->makeNode(at,temp.cval));
} else {
temp.mnh = expr->func->getMangledNameHash();
@@ -2624,12 +2637,12 @@ namespace das
// (e.g. struct field defaults) bypass folding, so the original ExprVar reference
// survives. Re-simulating it would emit a GetSharedMnh / GetGlobalMnh that looks
// up a mnh the variable lookup does not hold -> crash. Emit the const init directly instead.
- if ( expr->variable->index < 0 && expr->variable->init
+ if ( programIndexOf(context,expr->variable) < 0 && expr->variable->init
&& expr->variable->init->rtti_isConstant() ) {
setE(expr, simulateExpression(expr->variable->init));
return expr;
}
- DAS_ASSERT(expr->variable->index >= 0 && "using variable which is not used. how?");
+ DAS_ASSERT(programIndexOf(context,expr->variable) >= 0 && "using variable which is not used. how?");
uint64_t mnh = expr->variable->getMangledNameHash();
if ( !expr->variable->module->isSolidContext ) {
if ( expr->variable->global_shared ) {
@@ -2675,7 +2688,7 @@ namespace das
} else {
auto pCall = static_cast