From 4f757db8c0c5c87ba8d2772a374bcf879e8c3610 Mon Sep 17 00:00:00 2001 From: Churkin Aleksey Date: Wed, 9 Sep 2026 02:00:12 +0300 Subject: [PATCH 1/3] daslib: a standalone context speaks C++ and C, from one header The generated .cpp held a C++ class whose methods call the AOT functions directly. It now also emits one `extern "C"` entry point per export, wrapping the method beside it, plus an opaque handle and _create / _destroy / _last_error - so one pair serves a C++ host and a C host at once. daslib/c_api_header owns which signatures can cross, how each type is spelled and the layout asserts; aot_standalone owns the C++ spellings. Function.flags.exports is the only selection truth, which is what licenses skipping an unspellable signature with a warning while a function that ASKED for C and cannot cross is an error. The .cpp carries the header's text rather than including it: the header is the host's copy to move or edit, and the implementation must not break with it. The context namespace takes a ctx_ prefix - a name from the file stem lands inside namespace das, where cast.das redeclared das::cast - while the C prefix drops keyword escaping, since every emitted name suffixes it. Generated files are named for the input file, not the module it declares. The fixed API is defined even when nothing is exported, and what das owns on the heap crosses as an opaque handle a host hands back. --- .gitignore | 2 + daslib/ARCHITECTURE.md | 3 +- daslib/ARCHITECTURE_CAPI.md | 68 + daslib/ARCHITECTURE_EMIT.md | 22 + daslib/aot_standalone.das | 300 +++-- daslib/ast_boost.das | 12 + daslib/c_api_header.das | 1133 +++++++++++++++++ daslib/export_c.das | 49 + doc/source/reference/embedding/advanced.rst | 65 + doc/source/reference/language/annotations.rst | 34 + ...integration_cpp_20_standalone_contexts.rst | 4 +- examples/standalone/01_pure/main.cpp | 10 +- examples/standalone/02_heap/main.cpp | 2 +- examples/standalone/03_closures/main.cpp | 2 +- examples/standalone/04_c_binding/main.cpp | 2 +- .../standalone/05_compile_time_table/main.cpp | 2 +- examples/standalone/06_full_runtime/main.cpp | 2 +- examples/standalone/CMakeLists.txt | 2 + skills/cpp_integration.md | 52 +- .../daslang/references/modules-and-stdlib.md | 5 +- src/ast/ast_print.cpp | 2 +- src/builtin/module_builtin_ast_adapters.cpp | 14 + tests-cpp/big/nano_ctx/CMakeLists.txt | 1 + tests-cpp/big/nano_ctx/test_nano_ctx.cpp | 16 +- tests-cpp/big/standalone_ctx/CMakeLists.txt | 101 ++ .../standalone_ctx/expect_layout_assert.cmake | 29 + .../standalone_init_fixture.das | 32 + .../standalone_layout_fixture.das | 15 + .../test_standalone_bindings_host.das | 30 + .../big/standalone_ctx/test_standalone_capi.c | 65 + .../standalone_ctx/test_standalone_ctx.cpp | 11 +- .../test_standalone_layout_packed.c | 8 + .../test_standalone_modules.cpp | 4 +- tests/aot/test_standalone_emit.das | 20 +- .../integration/cpp/20_standalone_context.cpp | 2 +- utils/CMakeLists.txt | 1 + utils/aot/main.das | 8 +- utils/watchdog/main.cpp | 2 +- 38 files changed, 2007 insertions(+), 125 deletions(-) create mode 100644 daslib/ARCHITECTURE_CAPI.md create mode 100644 daslib/c_api_header.das create mode 100644 daslib/export_c.das create mode 100644 tests-cpp/big/standalone_ctx/expect_layout_assert.cmake create mode 100644 tests-cpp/big/standalone_ctx/standalone_layout_fixture.das create mode 100644 tests-cpp/big/standalone_ctx/test_standalone_bindings_host.das create mode 100644 tests-cpp/big/standalone_ctx/test_standalone_capi.c create mode 100644 tests-cpp/big/standalone_ctx/test_standalone_layout_packed.c diff --git a/.gitignore b/.gitignore index 6782e960da..d2bca6bc67 100644 --- a/.gitignore +++ b/.gitignore @@ -48,6 +48,8 @@ modules/**/*.shared_module examples/**/*.shared_module _aot_generated/ _llvm_aot_generated/ +# the -ctx driver writes each fixture's daslang bindings beside the host that requires them +tests-cpp/big/standalone_ctx/_*_c.das .vscode/ .cache/ diff --git a/daslib/ARCHITECTURE.md b/daslib/ARCHITECTURE.md index ae4c6e5510..6de07e7cda 100644 --- a/daslib/ARCHITECTURE.md +++ b/daslib/ARCHITECTURE.md @@ -3,10 +3,11 @@ Design rationale a maintainer cannot recover from the code alone. One numbered section per module; entries are anchored to symbols. -Three companions carry a concern each; a section number is unique across all four files. +Four companions carry a concern each; a section number is unique across all five files. - `ARCHITECTURE_LINT.md` - sec. 1-4: perf_lint, lint_config, lint, style_lint. - `ARCHITECTURE_EMIT.md` - sec. 5-7, 28-29: aot_cpp, aot_standalone, flatten, the shader rails. +- `ARCHITECTURE_CAPI.md` - sec. 30: c_api_header, the C surface both backends emit. - `ARCHITECTURE_LINQ.md` - sec. 11-17, 33, 37: the linq family, sql_linq, sql_migrate. ## 8. ast_verify diff --git a/daslib/ARCHITECTURE_CAPI.md b/daslib/ARCHITECTURE_CAPI.md new file mode 100644 index 0000000000..74fcd14c6a --- /dev/null +++ b/daslib/ARCHITECTURE_CAPI.md @@ -0,0 +1,68 @@ +# daslib architecture notes - the generated C API + +Companion to `ARCHITECTURE.md` in this folder; section numbers are unique across the family. +The header this section describes is the one a standalone context ships. + +## 30. c_api_header + +- **One module writes the generated header, and it owns the C declarations in it.** The C + declarations come first; the C++ half (`CppApi`) follows under `#ifdef __cplusplus`, and both + hosts include one file without seeing the other's half. `aot_standalone` emits the matching + `extern "C"` bodies off this describer, so the header and the source cannot disagree about what + crosses or what it is called. +- **C sits ON TOP of C++, not under it.** The C++ methods are DEFINED in the generated `.cpp` and + call the AOT functions directly; each `extern "C"` entry point is a wrapper over the method + beside it. So the C++ API is the wide one - every export, in native C++ types, spelled by + `aot_standalone` because it owns C++ types the way this module owns C ones - and C is the subset + C can express, not a ceiling over it. A method keeps the daslang name: `[export_c(name = ...)]` + renames the C symbol only. +- **The generated `.cpp` carries the header's text instead of including it.** The header is the + HOST's file - it may move it or edit it - and the implementation must not be breakable that way. + Types are declared once (`type_defs` never reaches the bodies), so inlining is a substitution for + the include rather than a second copy. A host header that drifts from the library still fails + loudly: a changed or removed signature is a link error, a changed layout trips the header's own + size and offset asserts. +- **`Function.flags.exports` is the whole selection truth.** `[export_c]` (`ExportCAnnotation`, + `daslib/export_c.das` - a das `[function_macro]`, so the C surface is decided entirely in + daslang) sets it, and so does `[export]`, so a standalone context takes the bit however it was + set. Accepting the bit however it was set is what licenses skipping an unspellable signature with + a warning; a function that ASKED for C with `[export_c]` and cannot cross is an error. No second + list can drift from the bit. Whether a signature CAN cross is decided here instead, after infer, + because argument types do not exist when an annotation applies. +- **Refusal is per stage, not per module**: `collect_c_exports` returns its rejections and + logs its skips. An `[export_c]` that cannot cross comes back for the caller to report - + `macro_error` during compilation, the jit error log during codegen - so this module needs + no `ProgramPtr` and no reporting policy of its own. A merely-public function is skipped + with a warning naming it and the type. +- **The scalar widths and the vector layouts are C++-side facts this emitter mirrors.** + `bool` is one byte (the `static_assert(sizeof(bool)==1)` in `getTypeBaseSize`, + `src/simulate/debug_info.cpp`), so das `bool` meets C as `bool`. `float3` is `{x, y, z}` at + 12 bytes and 4-byte alignment, because `vec3` (`include/daScript/misc/vectypes.h`) + is a plain three-field struct with no `alignas` - the 16-byte vec4f shape is the JIT's + register ABI, not the memory layout a header has to mirror. So no vector or structure carries + an alignment attribute, and every declared struct carries a size assert plus one offset assert + per field. `Structure.sizeOf` is already rounded to the struct's alignment + (`Structure::getSizeOf`, `src/ast/ast.cpp`), so `sizeof` in C matches it directly. +- **A bound value type crosses as the wrap type its annotation carries.** A + `ManagedValueAnnotation` is not a ref type and holds `makeValueType()` - the `WrapType::type` + das moves the value through, reachable from das as `get_underlying_value_type`. The header + typedefs that shape under the das type's name and the body assigns the das type through it; a C++ + host still gets the real type, because the C++ half carries the module's `aotRequire` include. It + is the one C type with an alignment attribute - a handle's alignment is not its wrap type's + (`BigEntityId` is 16-aligned, four floats are 4-aligned) - on ONE declarator, since the attribute + applies per declarator. Size and alignment are both asserted. A ref-type handle stays `void *`. +- **An enumeration is a typedef of its base integer plus loose enumerators, never a C + `enum`** - a C enum's underlying type is implementation-defined, which would break the + size assert on the 8/16/64-bit bases and on negative values. The values are read off the + entry's folded constant, so no smart pointer is needed to reach `find_enum_value`. +- **Types are emitted only when a signature reaches them, in post-order.** A by-value + field's structure is defined before the structure holding it; every structure also gets a + forward typedef ahead of all definitions, which is what lets a self-referential + (`Node?`) field compile. A pointer's target only has to be NAMEABLE, so one whose fields C + cannot spell is forward-declared and never DEFINED - defining it emits fields with no type at + all - while a representable target is defined, so a host can read through the pointer. A cycle + terminates either way, and a pointer to something C cannot even name degrades to `void *` + rather than refusing the function. +- **A `fixed_array` argument crosses (as `const T *`, which is what the das ABI already + passes) but a `fixed_array` result does not** - that would be a CMRES of an array, a + pointer nothing on the C side sizes. diff --git a/daslib/ARCHITECTURE_EMIT.md b/daslib/ARCHITECTURE_EMIT.md index cc91418de8..99cf48777a 100644 --- a/daslib/ARCHITECTURE_EMIT.md +++ b/daslib/ARCHITECTURE_EMIT.md @@ -52,8 +52,30 @@ Companion to `ARCHITECTURE.md` in this folder; section numbers are unique across dasHV takes `rtti_core`). A module missing from the daslib list still registers, only in the dependencies-first pass that follows; a module added to the C++ side joins the list. +- **The AnnotationInfo table resets at the START of the debug-info dump, not its end.** The + globals' `VarInfo`s are written after that dump and a handled global's info refers to an + `AnnotationInfo` by the name the dump minted, so clearing on the way out left `&` with nothing + after it. Only that walk reaches a global's annotation - `writeHandledAnnotations` iterates + types, structs and functions. +- **A member pointer is qualified with `aotModuleName`, never the raw module name.** The main + module is unnamed, so `_module.name` is empty for every type a script declares itself, while + `describeCppType` resolves those types through `g_aot_main_module_name`. Where one emitter writes + both - `das_safe_navigation` - they must agree, or the type argument names the + context's namespace while the member pointer names nothing. Only a standalone context sets a + main-module name, so regular AOT never sees it. + ## 6. aot_standalone +- **The entry module's structures are visited SORTED.** `visitModule` takes `sortStructures`, + which runs `topoSortStructures` so a by-value field's structure is complete before the structure + holding it; regular AOT passes it through `visit(program, adapter, true)` and a standalone + context, which visits the entry module by itself, has to ask for it too. Declaration order is the + author's, and nothing else re-derives it. +- **The context name is an identifier; the file stem is not the same string.** A stem reaches C++ + as a namespace and C as a symbol prefix, so `while.das` or `3d-math.das` would open + `namespace while {`. `context_name` is the stem through `cpp_context_ident`, while `file_stem` + keeps the raw name - a build system predicts the generated file names from the input path and + cannot be told they were sanitized. - **The generated constructor IS the init protocol** - a standalone context never calls `Context::runInitScript`, so the ctor reproduces its observable semantics inline: `memset(context.globals, 0, context.getGlobalSize())` mirrors runInitScript's globals diff --git a/daslib/aot_standalone.das b/daslib/aot_standalone.das index d1e4bfca6a..c1bb4d4cf6 100644 --- a/daslib/aot_standalone.das +++ b/daslib/aot_standalone.das @@ -15,24 +15,11 @@ require daslib/functional require daslib/ast_print_flags require daslib/aot_constants require daslib/aot_cpp +require daslib/c_api_header options strict_smart_pointers = false -struct StandaloneContextCfg { - //! Configuration for standalone context generation. - context_name : string; - class_name : string; - cpp_output_dir : string; - cross_platform : bool; - //! the context links C++ modules beyond the builtin one, so its constructor registers them - registers_modules : bool -}; - -def aotFunctionName(str : string) { - return replace(str, "`", "__") -} - def writeStandaloneContextMethods(var prog : ProgramPtr; var logs : StringBuilderWriter; prefix : string; declare_only : bool; cfg : StandaloneContextCfg) { let fnn = collectProgramUsedFunctions(prog, false, false); @@ -43,7 +30,7 @@ def writeStandaloneContextMethods(var prog : ProgramPtr; var logs : StringBuilde if (declare_only) { write(logs, " "); } - write(logs, "auto {prefix}{aotFunctionName(string(fn.origin != null ? fn.origin.name : fn.name))} ( "); + write(logs, "auto {prefix}{c_ident(standalone_function_name(fn))} ( "); var vars : array vars |> reserve(length(fn.arguments)) for (variable in fn.arguments) { @@ -143,6 +130,7 @@ def private makeGlobalVarInfos(program : ProgramPtr; var helper : AotDebugInfoHe def private writeGlobalVarInfos(var helper : AotDebugInfoHelper?; infos : array>; var tw : StringBuilderWriter) { for (entry in infos) { let suffix = "_gvar_{entry.index}" + helper->registerHandledAnnotation(unsafe(addr(tw)), entry.info) helper->writeDim(unsafe(addr(tw)), entry.info, suffix) helper->writeArgTypes(unsafe(addr(tw)), entry.info, suffix) helper->writeArgNames(unsafe(addr(tw)), entry.info, suffix) @@ -291,7 +279,7 @@ def writeStandaloneCtor(cfg : StandaloneContextCfg; initFunctions : string; var write(tw, " context.tabAdLookup = make_shared>();\n"); program.get_ptr() |> for_each_module_no_order($(pm) { pm |> for_each_annotation_ordered($(k; v) { - write(tw, " (*context.tabAdLookup)[{k : x}] = {v};\n"); + write(tw, " (*context.tabAdLookup)[{k:#x}] = {v};\n"); }); }); @@ -305,18 +293,8 @@ def writeStandaloneCtor(cfg : StandaloneContextCfg; initFunctions : string; var write(tw, "\}\n"); } -def writeStandaloneContext(var program : ProgramPtr, initFunctions : string, var header : StringBuilderWriter, var source : StringBuilderWriter; cfg : StandaloneContextCfg; globals : array; var context : Context) { - - write(header, "\n\n"); - //! the module scope is the FIRST base: constructed before Context, destroyed after it, so the - //! registry outlives the context's own teardown - let bases = cfg.registers_modules ? "public StandaloneModuleScope, public Context" : "public Context" - write(header, "class {cfg.class_name} : {bases} \{\n"); - write(header, "public:\n"); - write(header, " {cfg.class_name}();\n"); - writeStandaloneContextMethods(program, header, "", true, cfg); - write(header, "\};\n"); +def writeStandaloneContext(var program : ProgramPtr, initFunctions : string, var source : StringBuilderWriter; cfg : StandaloneContextCfg; globals : array; var context : Context) { writeStandaloneContextMethods(program, source, "{cfg.class_name}::", false, cfg); writeStandaloneCtor(cfg, initFunctions, source, program, globals, context); } @@ -378,11 +356,8 @@ class StandaloneContextGen : CppAot { }; -def writeModuleDeclarations(var header, source : StringBuilderWriter; registrations : array) { - //! the header carries the registry include (the class derives from its scope base); the - //! source carries the module declarations its registration table takes addresses of +def writeModuleDeclarations(var source : StringBuilderWriter; registrations : array) { if (empty(registrations)) return - write(header, "#include \"daScript/simulate/standalone_modules.h\"\n"); write(source, "#include \"daScript/simulate/standalone_modules.h\"\n"); for (entry in registrations) { write(source, "DECLARE_MODULE({entry.mod.cppClassName});\n"); @@ -402,8 +377,7 @@ def writeModuleRegistration(var source : StringBuilderWriter; registrations : ar write(source, "static const bool das_standalone_modules_added = standaloneAddModules(das_standalone_modules, {length(registrations)});\n\n"); } -def writeRegistration(var header : StringBuilderWriter; - var source : StringBuilderWriter; +def writeRegistration(var source : StringBuilderWriter; initFunctions : string; var program : ProgramPtr; cfg : StandaloneContextCfg; @@ -411,12 +385,10 @@ def writeRegistration(var header : StringBuilderWriter; globals : array; var context : Context) { write(source, "using namespace {program.thisNamespace};\n"); - write(header, "namespace {cfg.context_name} \{\n"); write(source, "namespace {cfg.context_name} \{\n"); dumpRegisterAot(unsafe(addr(source)), program, context, true, cfg.cross_platform); writeModuleRegistration(source, registrations); - writeStandaloneContext(program, initFunctions, header, source, cfg, globals, context); - write(header, "\} // namespace {cfg.context_name}\n"); + writeStandaloneContext(program, initFunctions, source, cfg, globals, context); write(source, "\} // namespace {cfg.context_name}\n"); } @@ -458,7 +430,6 @@ def addFunctionInfo(fnn : array; var helper : AotDebugInfoHelper?) { def genStandaloneSrc(var program : ProgramPtr; - var header : StringBuilderWriter; var source : StringBuilderWriter; cfg : StandaloneContextCfg; var coll : BlockVariableCollector?; globals : array) { var initFunctions : string; @@ -478,7 +449,7 @@ def genStandaloneSrc(var program : ProgramPtr; var gen = new StandaloneContextGen(program, unsafe(addr(tmp_writer)), coll, cfg.cross_platform); make_visitor(*gen) $(adapter) { gen.adapter := adapter - program |> visit_module(adapter, program.getThisModule); + program |> visit_module(adapter, program.getThisModule, true); // a generic instance several modules instantiated is one C++ function: emit the first copy only var emitted : table for (pfun in collectProgramUsedFunctions(program, false, false)) { @@ -505,20 +476,64 @@ def genStandaloneSrc(var program : ProgramPtr; } } } - write(header, type_defs); write(source, "namespace {program.thisNamespace} \{\n"); write(source, "{ctx_generated}"); write(source, "\} // namespace {program.thisNamespace}\n"); - return initFunctions + return <- (init_functions = initFunctions, type_defs = type_defs) +} + +def private moduleIsNoAot(var pmod : Module?) : bool { + var marked = 0 + var plain = 0 + pmod |> for_each_module_function() $(other) { + if (other.flags.builtIn) { + return + } + if (other.flags.noAot) { + marked++ + } else { + plain++ + } + } + return marked > 0 && plain == 0 +} + +def private whyNoAot(var pfun : Function?) : string { + if ((pfun |> find_annotation("no_aot")) != null) { + return "it is annotated [no_aot]" + } + if (moduleIsNoAot(pfun._module)) { + return "module {pfun._module.name} is marked no-AOT as a whole, which a module-level `options no_aot` does" + } + return "the compiler marked it no-AOT - it uses a construct the C++ emitter declines" +} + +def private declaredNoAot(program : ProgramPtr) : string { + var why = "" + program.get_ptr() |> for_each_module_no_order($(var pm) { + pm |> for_each_module_function() $(var pfun) { + if (!pfun.flags.noAot || !is_used(program, pfun)) { + return + } + if ((pfun |> find_annotation("no_aot")) == null && moduleIsNoAot(pm)) { + why = "it reaches {pm.name}, a module marked no-AOT as a whole" + } + } + }); + return why } -def private isMarkedNoAot(pfun : Function?) : bool { - for (ann in pfun.annotations) { - if (ann.annotation.name == "no_aot") { - return true +def private writeStandaloneSkipStub(program : ProgramPtr; cfg : StandaloneContextCfg; why : string) : bool { + let cur_mod = program.getThisModule.name.empty() ? cfg.file_stem : string(program.getThisModule.name) + to_log(LOG_WARNING, "standalone AOT skips {cur_mod}.das: {why}\n") + var ok = true + for (ext in ["h", "cpp"]) { + fopen("{cfg.cpp_output_dir}/{cfg.file_stem}.das.{ext}", "wb") $(fw) { + ok &&= fw != null + fwrite(fw, "// {cur_mod}.das has no standalone form: {why}\n") if (fw != null) } } - return false + return ok } def private checkAllUsedFunctionsCanAot(program : ProgramPtr) { @@ -528,11 +543,8 @@ def private checkAllUsedFunctionsCanAot(program : ProgramPtr) { return } if (pfun.flags.noAot) { - if (isMarkedNoAot(pfun)) { - macro_error(program, pfun.at, "standalone AOT cannot emit function {pfun.name}: it is [no_aot], and a standalone context has no interpreter") - } else { - macro_error(program, pfun.at, "standalone AOT cannot emit function {pfun.name}: it uses a type AOT cannot express, and a standalone context has no interpreter") - } + macro_error(program, pfun.at, "standalone AOT cannot emit function {pfun.name}: {whyNoAot(pfun)}, " + + "and a standalone context is C++ with no interpreted bodies to fall back to") } }); }); @@ -592,57 +604,163 @@ def private prepareProgramForEmission(var program : ProgramPtr; context : Contex return coll } +def private writeCApiBodies(var writer : StringBuilderWriter; var exports : array; names : CNames; cfg : StandaloneContextCfg) { + let ns = "das::{cfg.context_name}" + let api = "{names.prefix |> to_upper()}_API" + write(writer, "\n// C API - the entry points a C host links\n") + write(writer, "extern \"C\" \{\n") + write(writer, "{api} {names.prefix}_ctx * {names.prefix}_create(void) \{\n") + write(writer, " return ({names.prefix}_ctx *) new {ns}::{cfg.class_name}();\n\}\n") + write(writer, "{api} void {names.prefix}_destroy({names.prefix}_ctx * ctx) \{\n") + write(writer, " delete ({ns}::{cfg.class_name} *) ctx;\n\}\n") + write(writer, "{api} const char * {names.prefix}_last_error({names.prefix}_ctx * ctx) \{\n") + write(writer, " return ctx ? (({ns}::{cfg.class_name} *) ctx)->getException() : nullptr;\n\}\n") + write(writer, "{api} void {names.prefix}_shutdown_runtime(void) \{\n\}\n") + for (e in exports) { + var fn = e.fn + write(writer, "{c_declaration(e, names)} \{\n") + var args : array + args |> reserve(length(e.params) + 1) + for (p, variable in e.params, fn.arguments) { + let cpp = describeCppType(variable._type, DescribeConfig(skip_ref = true, cross_platform = cfg.cross_platform)) + if (p.by_pointer) { + args |> push("*({cpp} *) {p.name}") + } elif (variable._type.isString) { + args |> push("(char *) {p.name}") + } elif (variable._type.enumType != null || variable._type.isPointer) { + args |> push("({cpp}) {p.name}") + } else { + args |> push(p.name) + } + } + let arg_list = args |> join(", ") + let method = c_ident(standalone_function_name(fn)) + let call = "(({ns}::{cfg.class_name} *) ctx)->{method}({arg_list})" + if (e.result.via_out) { + let cpp_res = describeCppType(fn.result, DescribeConfig(skip_ref = true, cross_platform = cfg.cross_platform)) + write(writer, " *({cpp_res} *) out = {call};\n") + } elif (fn.result.isVoid) { + write(writer, " {call};\n") + } elif (fn.result.enumType != null || fn.result.isPointer) { + write(writer, " return ({e.result.c_type}) {call};\n") + } else { + write(writer, " return {call};\n") + } + write(writer, "\}\n") + } + write(writer, "\} // extern \"C\"\n") +} + + +def private writeDasBindings(var exports : array; names : CNames; + cfg : StandaloneContextCfg; cur_mod : string) : bool { + if (cfg.das_bindings_path |> empty()) { + return true + } + let lib = cfg.das_bindings_library + let b = DasBindings(linux_path = "{lib}.so", macos_path = "{lib}.dylib", windows_path = "{lib}.dll") + let by = "daslang utils/aot/main.das -- -ctx {cur_mod}.das {cfg.cpp_output_dir}" + return emit_das_bindings(exports, names, b, by, cfg.das_bindings_path) +} + +def private inlined_header(header, already : string) : string { + var seen : table + for (line in already |> split("\n")) { + if (line |> starts_with("#include")) { + seen |> insert(line) + } + } + return build_string() $(w) { + for (line in header |> split("\n")) { + if (line == "#pragma once" || ((line |> starts_with("#include")) && (seen |> key_exists(line)))) { + continue + } + write(w, "{line}\n") + } + } +} + + +def private writeStandaloneSource(mod_name, cur_mod : string; modules : array; + registrations : array; + header_content, bodies : string) : string { + return build_string() $(source) { + if (!empty(mod_name)) { + write(source, "// Module {mod_name}\n") + } + write(source, "#include \"daScript/simulate/standalone_ctx_utils.h\"\n") + write(source, "{join(modules, "")}") + writeModuleDeclarations(source, registrations) + write(source, "\n// {cur_mod}.das.h, inlined - the host compiles its own copy of this text\n") + write(source, inlined_header(header_content, join(modules, ""))) + write(source, "\n") + write(source, AOT_HEADERS) + write(source, bodies) + write(source, AOT_FOOTER) + } +} + + def public runStandaloneVisitor(var program : ProgramPtr, modules : array; registrations : array; var pctx : smart_ptr; cfg : StandaloneContextCfg) : bool { //! Runs the standalone AOT visitor on the program to generate C++ source and header files. //! `registrations` are the C++ modules the generated constructor registers, in order. //! Returns false (writing nothing) when emission collected errors. assume context = *pctx; + let no_aot = declaredNoAot(program) + if (!no_aot.empty()) { + return writeStandaloneSkipStub(program, cfg, no_aot) + } + var coll = prepareProgramForEmission(program, context) let globals <- usedGlobals(program) let mod = program.getThisModule; let mod_name = mod.moduleFlags.promoted ? "" : string(mod.name); set_aot_main_module_name(cfg.context_name) - var source_content = "" - let header_content = build_string() $(header) { - source_content = build_string() $(source) { - write(header, "#pragma once\n"); + let names = CNames(prefix = c_ident_part(cfg.file_stem), this_module = program.getThisModule) + var selected <- collect_c_exports(program.get_ptr(), names, true) + for (r in selected.errors) { + to_log(LOG_ERROR, "{r.message}\n") + } + var type_defs = "" + let bodies = build_string() $(source) { + write(source, "namespace das \{\n"); + let gen <- genStandaloneSrc(program, source, cfg, coll, globals); + type_defs = gen.type_defs + writeRegistration(source, gen.init_functions, program, cfg, registrations, globals, context); + writeCApiBodies(source, selected.exports, names, cfg); + write(source, "\} // namespace das\n"); + } + let cur_mod = mod.name.empty() ? cfg.file_stem : string(mod.name); + let cpp_includes = build_string() $(inc) { if (!empty(mod_name)) { - write(header, "// Module {mod_name}\n"); + write(inc, "// Module {mod_name}\n") } - write(header, "{AOT_INCLUDES}"); - write(header, "{join(modules, "")}"); - - if (!empty(mod_name)) { - write(source, "// Module {mod_name}\n"); + write(inc, "{AOT_INCLUDES}") + write(inc, "{join(modules, "")}") + if (!empty(registrations)) { + write(inc, "#include \"daScript/simulate/standalone_modules.h\"\n") } - - write(source, "#include \"daScript/simulate/standalone_ctx_utils.h\"\n"); - write(source, "{join(modules, "")}"); - writeModuleDeclarations(header, source, registrations); - write(source, "#include \"{mod.name.empty() ? cfg.context_name : string(mod.name)}.das.h\"\n\n"); - write(source, AOT_HEADERS); - - write(source, "namespace das \{\n"); - write(header, "namespace das \{\n"); - let initFunctions = genStandaloneSrc(program, header, source, cfg, coll, globals); - writeRegistration(header, source, initFunctions, program, cfg, registrations, globals, context); - write(source, "\} // namespace das\n"); - write(header, "\} // namespace das\n"); - - write(source, AOT_FOOTER); - } + let class_decl = build_string() $(decl) { + writeStandaloneContextMethods(program, decl, "", true, cfg) } + let cpp_api = CppApi(class_name = cfg.class_name, namespace_name = cfg.context_name, + includes = cpp_includes, type_defs = type_defs, class_decl = class_decl, + registers_modules = cfg.registers_modules) + let header_content = build_standalone_header(selected.exports, names, cpp_api, + "daslang utils/aot/main.das -- -ctx {cur_mod}.das {cfg.cpp_output_dir}", + "compile the generated .cpp and link it with the daslang runtime") + let source_content = writeStandaloneSource(mod_name, cur_mod, modules, registrations, header_content, bodies) set_aot_main_module_name("") - if (log_aot_emit_errors(program)) { + if (log_aot_emit_errors(program) || !(selected.errors |> empty()) + || !writeDasBindings(selected.exports, names, cfg, cur_mod)) { return false } - let cur_mod = mod.name.empty() ? cfg.context_name : string(mod.name); - let outputFile = "{cfg.cpp_output_dir}/{cur_mod}.das"; + let outputFile = "{cfg.cpp_output_dir}/{cfg.file_stem}.das"; fopen("{outputFile}.h", "wb") $(fw) { if (fw != null) { fwrite(fw, header_content) @@ -666,17 +784,35 @@ def public standalone_aot(input : string; output_dir : string; cross_platform : return standalone_aot(input, output_dir, cross_platform, paranoid_validation, cop) $(var _program : ProgramPtr) {} } +[export] +def public standalone_aot(input, output_dir, bindings_path, bindings_library : string; + cross_platform : bool, paranoid_validation : bool; cop : CodeOfPolicies) { + return standalone_aot(input, output_dir, bindings_path, bindings_library, + cross_platform, paranoid_validation, cop) $(var _program : ProgramPtr) {} +} + [export, unused_argument(paranoid_validation)] def public standalone_aot(input : string; output_dir : string; cross_platform : bool, paranoid_validation : bool; cop : CodeOfPolicies; before_emit : block<(var program : ProgramPtr) : void>) { + return standalone_aot(input, output_dir, "", "", cross_platform, paranoid_validation, cop, before_emit) +} + + +[export, unused_argument(paranoid_validation)] +def public standalone_aot(input, output_dir, bindings_path, bindings_library : string; + cross_platform : bool, paranoid_validation : bool; cop : CodeOfPolicies; + before_emit : block<(var program : ProgramPtr) : void>) { //! Compiles a daslang file and generates standalone AOT C++ code in the given output directory. //! `before_emit` receives the compiled program, so a driver can read annotations //! without compiling the program a second time. let file_name = input |> split_by_chars("/\\") |> back() - let ctx_name = (file_name |> split("."))[0] - var cfg = StandaloneContextCfg(context_name = ctx_name, + let ctx_stem = (file_name |> split("."))[0] + var cfg = StandaloneContextCfg(context_name = cpp_context_ident(ctx_stem), + file_stem = ctx_stem, class_name = "Standalone", cpp_output_dir = output_dir, - cross_platform = cross_platform) + cross_platform = cross_platform, + das_bindings_path = bindings_path, + das_bindings_library = bindings_library) var ok = false using() $(var mg : ModuleGroup) { var inscope access <- make_file_access("") diff --git a/daslib/ast_boost.das b/daslib/ast_boost.das index a300be9e42..5d98c7b153 100644 --- a/daslib/ast_boost.das +++ b/daslib/ast_boost.das @@ -925,6 +925,18 @@ def convert_to_expression(value : auto ==const) { return <- convert_to_expression(value, LineInfo()) } +def find_annotation(var fn : Function?; ann_name : string) : AnnotationDeclaration? { + if (fn == null) { + return null + } + for (ann in fn.annotations) { + if (ann != null && ann.annotation.name == ann_name) { + return ann + } + } + return null +} + def find_annotation(mod_name, ann_name : string) : Annotation const? { //! Finds an annotation by module name and annotation name in the compiling program. var mod = find_compiling_module(mod_name) diff --git a/daslib/c_api_header.das b/daslib/c_api_header.das new file mode 100644 index 0000000000..f0e905417c --- /dev/null +++ b/daslib/c_api_header.das @@ -0,0 +1,1133 @@ +options gen2 +options indenting = 4 + +module c_api_header shared private + +require daslib/ast_boost +require daslib/rtti +require daslib/fio +require daslib/strings_boost +require strings + + +let C_KEYWORDS <- {"restrict", "_Bool", "_Complex", "_Imaginary", "_Alignas", "_Alignof", + "_Atomic", "_Generic", "_Noreturn", "_Static_assert", "_Thread_local", "typeof"} + + +struct public StandaloneContextCfg { + context_name : string; + file_stem : string; + class_name : string; + cpp_output_dir : string; + cross_platform : bool; + registers_modules : bool; + das_bindings_path : string; + das_bindings_library : string +}; + + +def public c_ident_part(name : string) : string { + if (name |> empty()) { + return "" + } + var out = build_string() $(writer) { + name |> peek_data() $(bytes) { + for (b in bytes) { + let ch = int(b) + let keep = (is_alpha(ch) || is_number(ch) || ch == '_') + write_char(writer, keep ? ch : '_') + } + } + } + if (is_number(first_character(out))) { + out = "_{out}" + } + return out +} + + +def public c_ident(name : string) : string { + let out = c_ident_part(name) + return (is_cpp_keyword(out) || (C_KEYWORDS |> key_exists(out))) ? "{out}_" : out +} + + +def public cpp_context_ident(stem : string) : string { + let id = c_ident(stem) + return id |> empty() ? "das_ctx" : "ctx_{id}" +} + + +def public lib_prefix_from_path(input : string) : string { + let stem = input |> base_name() |> split(".") + return c_ident_part(stem |> empty() ? "" : stem[0]) +} + + +def public export_c_name_override(fn : Function?) : string { + for (ann in fn.annotations) { + if (ann.annotation.name != "export_c") { + continue + } + for (arg in ann.arguments) { + if (arg.name == "name" && arg.basicType == Type.tString) { + return string(arg.sValue) + } + } + } + return "" +} + + +def public has_export_c_annotation(var fn : Function?) : bool { + return (fn |> find_annotation("export_c")) != null +} + + +def private aotFunctionName(str : string) { + return replace(str, "`", "__") +} + +def public standalone_function_name(fn : Function?) : string { + let renamed = fn |> export_c_name_override() + if (!(renamed |> empty())) { + return renamed + } + return aotFunctionName(string(fn.origin != null ? fn.origin.name : fn.name)) +} + + +def public c_export_symbol(prefix : string; var fn : Function?; name_override : string) : string { + if (!(name_override |> empty())) { + return "{prefix}_{name_override}" + } + let das_name = string(fn.origin != null ? fn.origin.name : fn.name) + return "{prefix}_{c_ident_part(das_name)}" +} + + +struct public CType { + ok : bool + spelling : string + by_pointer : bool + opaque : bool +} + +struct CVector { + ok : bool + name : string + elem : string + lanes : int + span : bool + bytes : int +} + + +def private is_unsigned_type(t : Type) : bool { + return t == Type.tUInt || t == Type.tUInt8 || t == Type.tUInt16 || t == Type.tUInt64 +} + + +def private c_int_name(is_signed : bool; bytes : int) : string { + return "{is_signed ? "" : "u"}int{bytes * 8}_t" +} + + +def private scalar_of(t : TypeDeclPtr) : string { + if (t.isBool) { + return "bool" + } + if (t.isInteger) { + return c_int_name(t.isSignedInteger, t.sizeOf) + } + if (t.isBitfield) { + return c_int_name(false, t.sizeOf) + } + if (t.isFloatOrDouble) { + return t.sizeOf == 4 ? "float" : "double" + } + return "" +} + + +def private vector_of(var t : TypeDeclPtr) : CVector { + if (t == null || !t.isVectorType) { + return CVector(ok = false) + } + let lanes = t.vectorDim + let bytes = t.sizeOf / lanes + if (!t.isRange && (lanes < 2 || lanes > 4 || bytes != 4)) { + return CVector(ok = false) + } + let base = t.isRange ? t.rangeBaseType : t.vectorBaseType + let is_unsigned = is_unsigned_type(base) + var elem = c_int_name(!is_unsigned, bytes) + var name = "{is_unsigned ? "uint" : "int"}{lanes}" + if (base == Type.tFloat || base == Type.tDouble) { + elem = bytes == 4 ? "float" : "double" + name = "{elem}{lanes}" + } + if (t.isRange) { + name = "{is_unsigned ? "u" : ""}range{bytes == 8 ? "64" : ""}" + } + return CVector(ok = true, name = name, elem = elem, lanes = lanes, span = t.isRange, bytes = t.sizeOf) +} + + +let VEC_LANE_NAMES <- ["x", "y", "z", "w"] +let SPAN_LANE_NAMES <- ["from", "to"] + + +def private vector_field_names(v : CVector) : array { + return [for (i in range(v.lanes)); v.span ? SPAN_LANE_NAMES[i] : VEC_LANE_NAMES[i]] +} + + +struct public CNames { + prefix : string + @do_not_delete this_module : Module? +} + + +def private c_type_name(names : CNames; var owner : Module?; name : das_string) : string { + if (owner == null || owner == names.this_module || owner.name |> empty()) { + return "{names.prefix}_{c_ident_part(string(name))}" + } + return "{names.prefix}_{c_ident_part(string(owner.name))}_{c_ident_part(string(name))}" +} + + +def private peel(var t : TypeDeclPtr) : TypeDeclPtr { + if (t.baseType == Type.tDistinct && t.firstType != null) { + return peel(t.firstType) + } + return t +} + + +def private is_pod_struct(var st : Structure?) : bool { + return st != null && !st.flags.isClass && !(st.fields |> empty()) +} + + +def private pointee_ok(var t : TypeDeclPtr) : bool { + if (t == null) { + return true + } + var pt = peel(t) + if (pt.isStructure) { + return is_pod_struct(pt.structType) + } + if (pt.isEnumT) { + return pt.enumType != null && !pt.enumType.external + } + return !(scalar_of(pt) |> empty()) || vector_of(pt).ok || pt.isVoid +} + + +def private pointer_c_type(var t : TypeDeclPtr; names : CNames) : CType { + if (t.flags.smartPtr) { + return CType(ok = false) + } + var target = t.firstType + if (target == null || target.isVoid || !pointee_ok(target)) { + return CType(ok = true, spelling = "void *") + } + var pt = peel(target) + var inner = "" + if (pt.isStructure) { + inner = c_type_name(names, pt.structType._module, pt.structType.name) + } elif (pt.isEnumT) { + inner = c_type_name(names, pt.enumType._module, pt.enumType.name) + } else { + let v = vector_of(pt) + inner = v.ok ? "{names.prefix}_{v.name}" : scalar_of(pt) + } + let qual = pt.flags.constant ? "const " : "" + return CType(ok = true, spelling = "{qual}{inner} *") +} + + +def private field_c_type(var t : TypeDeclPtr; names : CNames; var visiting : table) : CType { + var ft = peel(t) + if (ft.baseType == Type.tFixedArray) { + let inner = ft.firstType == null ? CType(ok = false) : field_c_type(ft.firstType, names, visiting) + return inner.ok && !inner.by_pointer ? CType(ok = true, spelling = inner.spelling) : CType(ok = false) + } + if (ft.isStructure) { + if (!struct_c_ok(ft.structType, names, visiting)) { + return CType(ok = false) + } + return CType(ok = true, spelling = c_type_name(names, ft.structType._module, ft.structType.name)) + } + return value_c_type(ft, names) +} + + +def private struct_c_ok(var st : Structure?; names : CNames; var visiting : table) : bool { + if (!is_pod_struct(st)) { + return false + } + let key = "{st._module.name}::{st.name}" + if (visiting |> key_exists(key)) { + return true + } + visiting |> insert(key) + var ok = true + for (fld in st.fields) { + if (!field_c_type(fld._type, names, visiting).ok) { + ok = false + break + } + } + visiting |> erase(key) + return ok +} + + +def private handle_wrap_type(var t : TypeDeclPtr) : TypeDeclPtr { + if (t.baseType != Type.tHandle || t.annotation == null || t.annotation.isRefType) { + return TypeDeclPtr() + } + return <- get_underlying_value_type(t) +} + + +def private handle_c_type(var t : TypeDeclPtr; names : CNames) : CType { + var wrap <- handle_wrap_type(t) + if (wrap == null) { + return CType(ok = false) + } + let inner = value_c_type(wrap, names) + if (!inner.ok) { + return CType(ok = false) + } + return CType(ok = true, spelling = c_type_name(names, t.annotation._module, t.annotation.name), + by_pointer = inner.by_pointer) +} + + +def private value_c_type(var t : TypeDeclPtr; names : CNames) : CType { + if (t.baseType == Type.tHandle) { + return handle_c_type(t, names) + } + if (t.isPointer) { + return pointer_c_type(t, names) + } + if (t.isString) { + return CType(ok = true, spelling = "const char *") + } + if (t.isEnumT) { + if (t.enumType == null || t.enumType.external) { + return CType(ok = false) + } + return CType(ok = true, spelling = c_type_name(names, t.enumType._module, t.enumType.name)) + } + let v = vector_of(t) + if (v.ok) { + return CType(ok = true, spelling = "{names.prefix}_{v.name}", by_pointer = true) + } + let sc = scalar_of(t) + return sc |> empty() ? CType(ok = false) : CType(ok = true, spelling = sc) +} + + +def private opaque_c_of(var t : TypeDeclPtr; names : CNames) : CType { + let bt = t.baseType + let crosses = (bt == Type.tArray || bt == Type.tTable || bt == Type.tTuple || bt == Type.tVariant + || bt == Type.tIterator || bt == Type.tLambda + || (bt == Type.tStructure && t.structType != null) + || (bt == Type.tHandle && t.annotation != null)) + if (!crosses) { + return CType(ok = false) + } + let described = t.describe() |> replace(" const", "") + let spelling = "{names.prefix}_{c_ident_part(described)}" + return CType(ok = true, spelling = spelling, by_pointer = true, opaque = true) +} + + +def public c_type_of(var t : TypeDeclPtr; names : CNames; allow_opaque : bool = false) : CType { + if (t == null) { + return CType(ok = false) + } + var pt = peel(t) + if (pt.isVoid) { + return CType(ok = true, spelling = "void") + } + if (pt.isStructure) { + var visiting : table + if (!struct_c_ok(pt.structType, names, visiting)) { + return CType(ok = false) + } + return CType(ok = true, spelling = c_type_name(names, pt.structType._module, pt.structType.name), by_pointer = true) + } + if (pt.baseType == Type.tFixedArray) { + var visiting : table + let inner = pt.firstType == null ? CType(ok = false) : field_c_type(pt.firstType, names, visiting) + return inner.ok ? CType(ok = true, spelling = inner.spelling, by_pointer = true) : CType(ok = false) + } + let val = value_c_type(pt, names) + if (val.ok || !allow_opaque) { + return val + } + return opaque_c_of(pt, names) +} + + +struct public CParam { + name : string + c_type : string + by_pointer : bool + is_const : bool + pointer_in_impl : bool +} + + +struct public CResult { + c_type : string + via_out : bool + cmres : bool +} + + +struct public CExport { + @do_not_delete fn : Function? + c_name : string + das_signature : string + params : array + result : CResult +} + + +struct CReject { + ok : bool + what : string + why : string +} + + +def private param_name_of(raw : string; index : int) : string { + let n = c_ident(raw) + if (n |> empty()) { + return "a{index}" + } + return n == "ctx" || n == "out" ? "{n}_" : n +} + + +def private describe_params(var fn : Function?; names : CNames; var out : array) : CReject { + for (arg, i in fn.arguments, count()) { + let ct = c_type_of(arg._type, names, true) + if (!ct.ok || arg._type.isVoid) { + return CReject(what = "parameter `{arg.name}`", why = arg._type.describe()) + } + let by_ptr = ct.by_pointer || arg._type.flags.ref + out |> emplace(CParam(name = param_name_of(string(arg.name), i), + c_type = ct.spelling, + by_pointer = by_ptr, + is_const = by_ptr && arg._type.flags.constant, + pointer_in_impl = arg._type.isRef)) + } + return CReject(ok = true) +} + + +def private returns_cmres(var fn : Function?) : bool { + return fn.flags.copyOnReturn || fn.flags.moveOnReturn +} + + +def public describe_c_signature(var fn : Function?; names : CNames; c_name : string) : tuple { + var params : array + let bad = describe_params(fn, names, params) + if (!bad.ok) { + return (exp = CExport(), reject = bad) + } + let rt = c_type_of(fn.result, names) + if (!rt.ok || peel(fn.result).baseType == Type.tFixedArray) { + return (exp = CExport(), reject = CReject(what = "result", why = fn.result.describe())) + } + let res = CResult(c_type = rt.spelling, via_out = rt.by_pointer, cmres = returns_cmres(fn)) + return (exp = CExport(fn = fn, c_name = c_name, das_signature = das_signature_of(fn), + params <- params, result = res), reject = CReject(ok = true)) +} + + +def private das_signature_of(var fn : Function?) : string { + let args = [for (a in fn.arguments); "{a.name} : {a._type.describe() |> replace(" const", "")}"] + return "def {fn.name}({args |> join("; ")}) : {fn.result.describe() |> replace(" const", "")}" +} + + +struct CEnum { + @do_not_delete en : Enumeration? + base : string +} + + +struct CHandle { + c_name : string + cpp_name : string + wrap : string + lanes : int + bytes : int + align : int + by_pointer : bool +} +struct CTypeSet { + enums : array + vectors : array + handles : array + @do_not_delete structs : array + @do_not_delete opaque : array + handle_names : array + seen : table +} + + +def private collect_type(var t : TypeDeclPtr; names : CNames; var set : CTypeSet) { + if (t == null) { + return + } + var pt = peel(t) + if (pt.isPointer) { + var pointee = pt.firstType == null ? TypeDeclPtr() : peel(pt.firstType) + if (pointee != null && pointee.isStructure) { + var visiting : table + if (struct_c_ok(pointee.structType, names, visiting)) { + collect_struct(pointee.structType, names, set) + } else { + collect_opaque_struct(pointee.structType, set) + } + } else { + collect_type(pt.firstType, names, set) + } + return + } + if (pt.baseType == Type.tFixedArray) { + collect_type(pt.firstType, names, set) + return + } + if (pt.isEnumT && pt.enumType != null && !pt.enumType.external) { + let key = "e:{pt.enumType._module.name}::{pt.enumType.name}" + if (!(set.seen |> key_exists(key))) { + set.seen |> insert(key) + set.enums |> push(CEnum(en = pt.enumType, + base = c_int_name(!is_unsigned_type(pt.enumType.baseType), pt.sizeOf))) + } + return + } + let v = vector_of(pt) + if (v.ok) { + let key = "v:{v.name}" + if (!(set.seen |> key_exists(key))) { + set.seen |> insert(key) + set.vectors |> push(v) + } + return + } + if (pt.baseType == Type.tHandle) { + collect_handle(pt, names, set) + return + } + if (pt.isStructure) { + collect_struct(pt.structType, names, set) + return + } + collect_opaque_tag(pt, names, set) +} + + +def private collect_opaque_tag(var t : TypeDeclPtr; names : CNames; var set : CTypeSet) { + let op = opaque_c_of(t, names) + if (op.ok && !(set.seen |> key_exists("o:{op.spelling}"))) { + set.seen |> insert("o:{op.spelling}") + set.handle_names |> push(op.spelling) + } +} + + +def private collect_handle(var t : TypeDeclPtr; names : CNames; var set : CTypeSet) { + var wrap <- handle_wrap_type(t) + let inner = wrap == null ? CType(ok = false) : value_c_type(wrap, names) + if (!inner.ok) { + collect_opaque_tag(t, names, set) + return + } + let key = "h:{t.annotation._module.name}::{t.annotation.name}" + if (set.seen |> key_exists(key)) { + return + } + set.seen |> insert(key) + let v = vector_of(wrap) + set.handles |> push(CHandle( + c_name = c_type_name(names, t.annotation._module, t.annotation.name), + cpp_name = string(t.annotation.cppName |> empty() ? t.annotation.name : t.annotation.cppName), + wrap = v.ok ? v.elem : inner.spelling, lanes = v.ok ? v.lanes : 0, + bytes = t.sizeOf, align = t.alignOf, by_pointer = inner.by_pointer)) +} + + +def private collect_opaque_struct(var st : Structure?; var set : CTypeSet) { + if (st == null) { + return + } + let key = "o:{st._module.name}::{st.name}" + if (set.seen |> key_exists(key)) { + return + } + set.seen |> insert(key) + set.opaque |> push(st) +} + + +def private collect_struct(var st : Structure?; names : CNames; var set : CTypeSet) { + if (st == null) { + return + } + let key = "s:{st._module.name}::{st.name}" + if (set.seen |> key_exists(key)) { + return + } + set.seen |> insert(key) + for (fld in st.fields) { + collect_type(fld._type, names, set) + } + set.structs |> push(st) +} + + +def public collect_c_types(var exports : array; names : CNames) : CTypeSet { + var set : CTypeSet + for (e in exports) { + for (arg in e.fn.arguments) { + collect_type(arg._type, names, set) + } + collect_type(e.fn.result, names, set) + } + return <- set +} + + +def private enum_value_of(var en : Enumeration?; name : das_string; base : string) : string { + let v = en |> find_enum_value(string(name)) + let is_unsigned = base |> starts_with("u") + let digits = is_unsigned ? "{uint64(v):d}" : "{v:d}" + if (base == "int64_t" || base == "uint64_t") { + return "{is_unsigned ? "UINT64_C" : "INT64_C"}({digits})" + } + return is_unsigned ? "{digits}u" : digits +} + + +def private write_enum(var writer : StringBuilderWriter; var e : CEnum; names : CNames) { + var en = e.en + let cname = c_type_name(names, en._module, en.name) + write(writer, "\n// das: enum {en.name}\ntypedef {e.base} {cname};\n") + write(writer, "enum \{\n") + let entries = [for (ee in en.list); " {cname}_{c_ident_part(string(ee.name))} = {enum_value_of(en, ee.name, e.base)}"] + write(writer, "{entries |> join(",\n")}\n\};\n") +} + + +def private write_handle(var writer : StringBuilderWriter; h : CHandle; names : CNames) { + let guard = names.prefix |> to_upper() + let body = h.lanes == 0 ? h.wrap : "struct \{ {guard}_ALIGNAS({h.align}) {h.wrap} lanes[{h.lanes}]; \}" + write(writer, "\n// das: {h.cpp_name} - a bound value type, carried as its ABI wrap type. Pass it\n") + write(writer, "// back as you received it; the lanes are the ABI's, not the type's own fields.\ntypedef {body} {h.c_name};\n") + write(writer, "{guard}_STATIC_ASSERT(sizeof({h.c_name}) == {h.bytes}, \"{h.c_name}: size differs from the daslang layout\");\n") + write(writer, "{guard}_STATIC_ASSERT({guard}_ALIGNOF({h.c_name}) == {h.align}, \"{h.c_name}: alignment differs from the daslang layout\");\n") +} + + +def private write_vector(var writer : StringBuilderWriter; v : CVector; names : CNames) { + let cname = "{names.prefix}_{v.name}" + let fields = vector_field_names(v) |> join(", ") + write(writer, "\n// das: {v.name}\ntypedef struct \{ {v.elem} {fields}; \} {cname};\n") + write(writer, "{names.prefix |> to_upper()}_STATIC_ASSERT(sizeof({cname}) == {v.bytes}, \"{cname}: size differs from the daslang layout\");\n") +} + + +def private field_decl(var fld : FieldDeclaration; names : CNames) : string { + var visiting : table + var ft = peel(fld._type) + var dims : array + while (ft.baseType == Type.tFixedArray && ft.firstType != null) { + dims |> push("[{ft.fixedDim}]") + ft = peel(ft.firstType) + } + let ct = field_c_type(fld._type, names, visiting) + return " {ct.spelling} {c_ident(string(fld.name))}{dims |> join("")};" +} + + +def private write_struct(var writer : StringBuilderWriter; var st : Structure?; names : CNames) { + let cname = c_type_name(names, st._module, st.name) + let guard = names.prefix |> to_upper() + write(writer, "\n// das: struct {st.name}\nstruct {cname} \{\n") + for (fld in st.fields) { + write(writer, "{field_decl(fld, names)}\n") + } + write(writer, "\};\n") + write(writer, "{guard}_STATIC_ASSERT(sizeof({cname}) == {st.sizeOf}, \"{cname}: size differs from the daslang layout\");\n") + for (fld in st.fields) { + write(writer, "{guard}_STATIC_ASSERT(offsetof({cname}, {c_ident(string(fld.name))}) == {fld.offset}, \"{cname}.{fld.name}: offset differs from the daslang layout\");\n") + } +} + + +def private param_decl(p : CParam) : string { + if (!p.by_pointer) { + return "{p.c_type} {p.name}" + } + return "{p.is_const ? "const " : ""}{p.c_type} * {p.name}" +} + + +def public c_declaration(e : CExport; names : CNames) : string { + var args = ["{names.prefix}_ctx * ctx"] + args |> reserve(length(e.params) + 2) + for (p in e.params) { + args |> push(param_decl(p)) + } + if (e.result.via_out) { + args |> push("{e.result.c_type} * out") + } + let ret = e.result.via_out ? "void" : e.result.c_type + let arg_list = args |> join(", ") + return "{names.prefix |> to_upper()}_API {ret} {e.c_name}({arg_list})" +} + + +def private write_prologue(var writer : StringBuilderWriter; names : CNames; generated_by : string) { + let guard = names.prefix |> to_upper() + write(writer, "// Code generated by `{generated_by}`. DO NOT EDIT.\n") + write(writer, "// Layouts are asserted for the daslang that generated this header; regenerate it for another target.\n") + write(writer, "#pragma once\n\n#include \n#include \n#include \n\n") + write(writer, "#if defined(__cplusplus)\n#define {guard}_STATIC_ASSERT(cond, msg) static_assert(cond, msg)\n") + write(writer, "#elif defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L\n") + write(writer, "#define {guard}_STATIC_ASSERT(cond, msg) _Static_assert(cond, msg)\n") + write(writer, "#else\n#define {guard}_STATIC_ASSERT(cond, msg)\n#endif\n\n") + write(writer, "// Building the generated .cpp into a Windows DLL exports nothing unless the entry points say\n") + write(writer, "// so: define {guard}_SHARED on both sides, and {guard}_BUILD on the library's own\n") + write(writer, "// translation unit. Everywhere else, and for a static link, both expand to nothing.\n") + write(writer, "#if defined(_WIN32) && defined({guard}_SHARED)\n") + write(writer, "#if defined({guard}_BUILD)\n#define {guard}_API __declspec(dllexport)\n") + write(writer, "#else\n#define {guard}_API __declspec(dllimport)\n#endif\n") + write(writer, "#else\n#define {guard}_API\n#endif\n\n") + write(writer, "#if defined(__cplusplus)\n#define {guard}_ALIGNAS(n) alignas(n)\n") + write(writer, "#elif defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L\n") + write(writer, "#define {guard}_ALIGNAS(n) _Alignas(n)\n") + write(writer, "#elif defined(_MSC_VER)\n#define {guard}_ALIGNAS(n) __declspec(align(n))\n") + write(writer, "#else\n#define {guard}_ALIGNAS(n) __attribute__((aligned(n)))\n#endif\n") + write(writer, "#if defined(__cplusplus)\n#define {guard}_ALIGNOF(t) alignof(t)\n") + write(writer, "#elif defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L\n") + write(writer, "#define {guard}_ALIGNOF(t) _Alignof(t)\n") + write(writer, "#else\n#define {guard}_ALIGNOF(t) __alignof__(t)\n#endif\n\n") + write(writer, "#ifdef __cplusplus\nextern \"C\" \{\n#endif\n") +} + + +def private write_fixed_api(var writer : StringBuilderWriter; names : CNames) { + let p = names.prefix + let guard = p |> to_upper() + write(writer, "\n// One library instance and everything it owns: globals, heap, string heap. Drive one\n") + write(writer, "// instance from one thread at a time, or make one per thread.\n") + write(writer, "typedef struct {p}_ctx {p}_ctx;\n") + write(writer, "\n// Creates an instance, then runs the script's global initializers and its [init] functions.\n") + write(writer, "// Callable from any thread, and from a process that already carries a daslang runtime -\n") + write(writer, "// another such library, or a host that registered the daslang modules itself. Returns NULL\n") + write(writer, "// when the initializers raised; {p}_last_error(NULL) reports why.\n") + write(writer, "{guard}_API {p}_ctx * {p}_create(void);\n") + write(writer, "\n// Runs the script's [finalize] functions, then frees the instance and everything it\n") + write(writer, "// allocated - every string and every pointer it returned included.\n") + write(writer, "{guard}_API void {p}_destroy({p}_ctx * ctx);\n") + write(writer, "\n// NULL when the last call on `ctx` completed normally; otherwise that call's exception text,\n") + write(writer, "// owned by the instance and valid until the next call on it. Pass NULL to read a failed\n") + write(writer, "// {p}_create. On a failed call the returned value is 0/false/NULL and `out` is left untouched.\n") + write(writer, "{guard}_API const char * {p}_last_error({p}_ctx * ctx);\n") + write(writer, "\n// Drains the daslang runtime for the whole process. Optional, and final: no instance of any\n") + write(writer, "// daslang library may be created or called afterwards.\n") + write(writer, "{guard}_API void {p}_shutdown_runtime(void);\n") + write(writer, "\n// A `const char *` an exported function returns points into the instance's string heap, and a\n") + write(writer, "// returned `T *` into its object heap: copy what you need past the next call on that instance,\n") + write(writer, "// because a later call may collect. The empty daslang string comes back NULL. A pointer you\n") + write(writer, "// pass in, `const char *` included, is borrowed for the duration of the call.\n") +} + + +def private write_c_section(var writer : StringBuilderWriter; var exports : array; names : CNames; + generated_by, link_note : string) { + var types <- collect_c_types(exports, names) + write_prologue(writer, names, generated_by) + write_fixed_api(writer, names) + for (en in types.enums) { + write_enum(writer, en, names) + } + for (vt in types.vectors) { + write_vector(writer, vt, names) + } + for (h in types.handles) { + write_handle(writer, h, names) + } + var defined : table + for (st in types.structs) { + defined |> insert(c_type_name(names, st._module, st.name)) + } + if (!(types.handle_names |> empty())) { + write(writer, "\n// values only daslang can build: hold the pointer, hand it back, never read through it\n") + for (h in types.handle_names) { + write(writer, "typedef struct {h} {h};\n") + } + } + if (!(types.structs |> empty()) || !(types.opaque |> empty())) { + write(writer, "\n") + for (st in types.structs) { + let cname = c_type_name(names, st._module, st.name) + write(writer, "typedef struct {cname} {cname};\n") + } + for (st in types.opaque) { + let cname = c_type_name(names, st._module, st.name) + if (!(defined |> key_exists(cname))) { + write(writer, "typedef struct {cname} {cname};\n") + } + } + } + for (st in types.structs) { + write_struct(writer, st, names) + } + write(writer, "\n") + for (e in exports) { + write(writer, "\n// {e.das_signature}\n{c_declaration(e, names)};\n") + } + if (!(link_note |> empty())) { + write(writer, "\n// Linking:\n") + for (line in link_note |> split("\n")) { + write(writer, "// {line}\n") + } + } + write(writer, "\n#ifdef __cplusplus\n\} // extern \"C\"\n#endif\n") +} + + +def public build_c_header(var exports : array; names : CNames; input_path, output_path, link_note : string) : string { + return build_string() $(var writer) { + write_c_section(writer, exports, names, "daslang -lib {input_path} -output {output_path}", link_note) + } +} + + +def public emit_c_header(var exports : array; names : CNames; input_path, output_path, link_note : string) : bool { + let text = build_c_header(exports, names, input_path, output_path, link_note) + let path = "{output_path}.h" + var ok = false + fopen(path, "wb") $(f) { + if (f != null) { + f |> fwrite(text) + ok = true + } + } + if (!ok) { + to_log(LOG_ERROR, "daslang -lib: can't write the C header {path}\n") + } + return ok +} + + +struct public CRejection { + fn_name : string + message : string +} + + +def public is_c_export_candidate(var fn : Function?; prog : Program?; export_all : bool) : bool { + if (fn._module != prog.getThisModule || !fn.flags.exports + || fn.flags.builtIn || fn.flags.generated || fn.moreFlags.isTemplate || fn.fromGeneric != null + || fn.flags.init || fn.flags.shutdown || fn.flags.macroInit || fn.moreFlags.macroFunction) { + return false + } + return export_all || has_export_c_annotation(fn) +} + + +def private reserved_c_names(names : CNames) : array { + let p = names.prefix + return ["{p}_create", "{p}_destroy", "{p}_last_error", "{p}_shutdown_runtime", "{p}_ctx"] +} + + +def public collect_c_exports(prog : Program?; names : CNames; export_all : bool) : tuple; errors : array> { + var accepted : array + var errors : array + var claimed <- {for (r in reserved_c_names(names)); r => "the fixed library API"} + var this_mod = prog.getThisModule + this_mod |> for_each_function("") $(var fn) { + if (!is_c_export_candidate(fn, prog, export_all)) { + return + } + let is_explicit = has_export_c_annotation(fn) + let c_name = c_export_symbol(names.prefix, fn, export_c_name_override(fn)) + var why = "" + let owner = claimed?[c_name] ?? "" + var sig <- describe_c_signature(fn, names, c_name) + if (!(owner |> empty())) { + why = "C symbol `{c_name}` is already taken by {owner}; C has no overloading - give one of them [export_c(name = \"...\")]" + } elif (!sig.reject.ok) { + why = ("{sig.reject.what} has type {sig.reject.why}, which has no C representation " + + "(allowed: bool, int8..uint64, int, uint, float, double, string, pointers, enums, " + + "POD structs, bound value types, float2..uint4, range/urange/range64/urange64, " + + "fixed_array arguments)") + } + if (!(why |> empty())) { + if (is_explicit) { + errors |> emplace(CRejection(fn_name = string(fn.name), + message = "{describe(fn.at)}: [export_c] {fn.name}: {why}")) + } else { + to_log(LOG_WARNING, "{describe(fn.at)}: skipping {fn.name} in the C API: {why}\n") + } + return + } + claimed[c_name] = "{fn.name} at {describe(fn.at)}" + accepted |> emplace(sig.exp) + } + return (exports <- accepted, errors <- errors) +} + + +struct public CppApi { + class_name : string + namespace_name : string + includes : string + type_defs : string + class_decl : string + registers_modules : bool +} + + +def private write_cpp_section(var writer : StringBuilderWriter; cpp : CppApi) { + write(writer, "\n#ifdef __cplusplus\n\n") + write(writer, cpp.includes) + write(writer, "namespace das \{\n") + write(writer, cpp.type_defs) + write(writer, "namespace {cpp.namespace_name} \{\n\n") + let bases = cpp.registers_modules ? "public StandaloneModuleScope, public Context" : "public Context" + write(writer, "class {cpp.class_name} : {bases} \{\npublic:\n {cpp.class_name}();\n") + write(writer, cpp.class_decl) + write(writer, "\};\n\n") + write(writer, "\} // namespace {cpp.namespace_name}\n\} // namespace das\n\n#endif // __cplusplus\n") +} + + +def public build_standalone_header(var exports : array; names : CNames; cpp : CppApi; + generated_by, link_note : string) : string { + return build_string() $(var writer) { + write_c_section(writer, exports, names, generated_by, link_note) + write_cpp_section(writer, cpp) + } +} + + +struct public DasBindings { + linux_path : string + macos_path : string + windows_path : string +} + + +def private das_scalar_of(var t : TypeDeclPtr) : string { + if (t.isBool) { + return "bool" + } + if (t.isFloatOrDouble) { + return t.sizeOf == 4 ? "float" : "double" + } + if (t.isInteger || t.isBitfield) { + let bits = t.sizeOf * 8 + let sign = t.isBitfield || !t.isSignedInteger ? "u" : "" + return bits == 32 ? "{sign}int" : "{sign}int{bits}" + } + return "" +} + + +def private das_struct_ok(var st : Structure?; names : CNames) : bool { + if (st == null) { + return false + } + for (fld in st.fields) { + if (das_field_of(fld, names) |> empty()) { + return false + } + } + return true +} + + +def private das_value_type(var raw : TypeDeclPtr; names : CNames) : string { + var t = peel(raw) + if (t.isString) { + return "string" + } + if (t.isEnumT) { + return t.enumType == null || t.enumType.external ? "" : "int" + } + if (t.baseType == Type.tHandle) { + return "" + } + if (t.isPointer) { + return "void?" + } + let v = vector_of(t) + if (v.ok) { + return v.name + } + if (t.isStructure) { + if (!das_struct_ok(t.structType, names)) { + return "" + } + return c_type_name(names, t.structType._module, t.structType.name) + } + return das_scalar_of(t) +} + + +def private das_field_of(var fld : FieldDeclaration; names : CNames) : string { + var t = peel(fld._type) + var dims : array + while (t.baseType == Type.tFixedArray && t.firstType != null) { + dims |> push("[{t.fixedDim}]") + t = peel(t.firstType) + } + if (length(dims) > 1) { + return "" + } + let base = das_value_type(t, names) + let suffix = dims |> join("") + return base |> empty() ? "" : "{base}{suffix}" +} + + +def private das_param_of(p : CParam; var raw : TypeDeclPtr; names : CNames) : string { + var t = peel(raw) + if (t.baseType == Type.tFixedArray && t.firstType != null) { + let elem = das_value_type(t.firstType, names) + return elem |> empty() ? "" : "{elem}?#" + } + let base = das_value_type(t, names) + if (base |> empty()) { + return "" + } + return p.by_pointer ? "{base}?#" : base +} + + +def private dasbind_can_wrap(float_at : array) : bool { + if (length(float_at) <= 6) { + return true + } + for (i in range(6, length(float_at))) { + if (float_at[i]) { + return false + } + } + return true +} + + +def private das_extern_signature(var e : CExport; names : CNames) : tuple { + var fn = e.fn + var args = ["ctx : void?"] + var float_at = [false] + args |> reserve(length(e.params) + 2) + for (pp, variable in e.params, fn.arguments) { + let spell = das_param_of(pp, variable._type, names) + if (spell |> empty()) { + let what = describe(variable._type) + return (sig = "", why = "argument `{pp.name}` is {what}") + } + args |> push("{pp.name} : {spell}") + float_at |> push(spell == "float" || spell == "double") + } + var result = "void" + if (e.result.via_out) { + let out_type = das_value_type(fn.result, names) + if (out_type |> empty()) { + let what = describe(fn.result) + return (sig = "", why = "result is {what}") + } + args |> push("out : {out_type}?#") + float_at |> push(false) + } elif (!fn.result.isVoid) { + result = das_value_type(fn.result, names) + if (result |> empty()) { + let what = describe(fn.result) + return (sig = "", why = "result is {what}") + } + } + if (!dasbind_can_wrap(float_at)) { + return (sig = "", why = "dasbind has no SysV wrapper for a floating-point argument past the sixth") + } + let arg_list = args |> join("; ") + return (sig = "{e.c_name}({arg_list}) : {result}", why = "") +} + + +def private write_extern(var writer : StringBuilderWriter; b : DasBindings; sym, sig : string) { + write(writer, "[extern(cdecl, late, name=\"{sym}\",\n") + write(writer, " linux_library=\"{b.linux_path}\",\n") + write(writer, " macos_library=\"{b.macos_path}\",\n") + write(writer, " windows_library=\"{b.windows_path}\")]\n") + write(writer, "def {sig} \{\}\n\n") +} + + +def public build_das_bindings(var exports : array; names : CNames; b : DasBindings; + generated_by : string) : string { + var types <- collect_c_types(exports, names) + let p = names.prefix + return build_string() $(var writer) { + write(writer, "options gen2\n\n// Code generated by `{generated_by}`. DO NOT EDIT.\n\n") + write(writer, "require dasbind public\n\n") + for (st in types.structs) { + if (!das_struct_ok(st, names)) { + continue + } + write(writer, "struct {c_type_name(names, st._module, st.name)} \{\n") + for (fld in st.fields) { + write(writer, " {c_ident(string(fld.name))} : {das_field_of(fld, names)}\n") + } + write(writer, "\}\n\n") + } + write_extern(writer, b, "{p}_create", "{p}_create() : void?") + write_extern(writer, b, "{p}_destroy", "{p}_destroy(ctx : void?) : void") + write_extern(writer, b, "{p}_last_error", "{p}_last_error(ctx : void?) : string") + write_extern(writer, b, "{p}_shutdown_runtime", "{p}_shutdown_runtime() : void") + for (e in exports) { + let spelled = das_extern_signature(e, names) + if (spelled.sig |> empty()) { + write(writer, "// {e.c_name} is declared in the C header only: {spelled.why}\n\n") + continue + } + write(writer, "// das: {e.das_signature}\n") + write_extern(writer, b, e.c_name, spelled.sig) + } + } +} + + +def public emit_das_bindings(var exports : array; names : CNames; b : DasBindings; + generated_by, path : string) : bool { + let text = build_das_bindings(exports, names, b, generated_by) + var ok = false + fopen(path, "wb") $(f) { + if (f != null) { + f |> fwrite(text) + ok = true + } + } + if (!ok) { + to_log(LOG_ERROR, "c_api_header: can't write the daslang bindings {path}\n") + } + return ok +} diff --git a/daslib/export_c.das b/daslib/export_c.das new file mode 100644 index 0000000000..65d68a7726 --- /dev/null +++ b/daslib/export_c.das @@ -0,0 +1,49 @@ +options gen2 +options indenting = 4 + +module export_c shared private !inscope + +require daslib/ast_boost +require strings + + +def private is_c_identifier(name : string) : bool { + if (name |> empty()) { + return false + } + var first = true + for (ch in name) { + let alpha = is_alpha(ch) || ch == '_' + if (!(alpha || (is_number(ch) && !first))) { + return false + } + first = false + } + return true +} + + +[function_macro(name = "export_c")] +class private ExportCAnnotation : AstFunctionAnnotation { + def override apply(var func : FunctionPtr; var group : ModuleGroup; + args : AnnotationArgumentList; var errors : das_string) : bool { + for (arg in func.arguments) { + if (arg._type.baseType == Type.autoinfer && arg.init == null) { + errors := "[export_c] can't export generic function `{func.name}`: it has auto or template arguments, and C needs one concrete signature" + return false + } + } + for (arg in args) { + if (arg.name != "name") { + errors := "[export_c] unknown argument `{arg.name}`; the only argument is name=\"\"" + return false + } + if (arg.basicType != Type.tString || !is_c_identifier(string(arg.sValue))) { + errors := "[export_c] name must be a valid C identifier ([A-Za-z_][A-Za-z0-9_]*), got `{arg.sValue}`" + return false + } + } + func.flags.exports = true + return true + } +} diff --git a/doc/source/reference/embedding/advanced.rst b/doc/source/reference/embedding/advanced.rst index 2058b2ced3..3ff5ff9dd2 100644 --- a/doc/source/reference/embedding/advanced.rst +++ b/doc/source/reference/embedding/advanced.rst @@ -4,6 +4,7 @@ .. index:: single: Embedding; AOT single: Embedding; Advanced Topics + single: Embedding; C Libraries single: Embedding; Class Adapters single: Embedding; Coroutines single: Embedding; Standalone Contexts @@ -270,10 +271,74 @@ Pipeline: das::standalone::Standalone ctx; ctx.test(); // direct call, no findFunction needed +The same generated files also carry a C API: one ``extern "C"`` entry +point per export, wrapping the C++ method beside it, plus an opaque +handle and ``_create`` / ``_destroy`` / ``_last_error``. So one +generated pair serves a C++ host and a C host at once, and the two +surfaces are not identical — C++ reaches every export in native types, +while C carries the subset C can express (the emitter names the ones it +skips). The ``.das.cpp`` embeds the header's text rather than including +it, so moving or editing your copy of the header cannot break the +library's own translation unit. + See :ref:`tutorial_integration_cpp_standalone_contexts` for a complete example. +C libraries +=========== + +``daslang -lib`` compiles a daslang program to a native library with a C +API, so a host that only *calls* one script needs no daslang headers, no +``Module``, and no compiler. The LLVM backend emits the library; a +standalone context (above) emits C++ source your build compiles instead. + +Pipeline: + +1. ``daslang -lib script.das -output build/script`` +2. This writes ``build/script.so`` (``.dylib`` / ``.dll``) and + ``build/script.h``. Add ``-- --jit-lib-static`` for a ``.a`` / ``.lib`` + archive instead; the header records what a host then links. +3. Include the header, link the library, call it. + +Which functions cross the boundary is your choice: ``[export_c]`` marks +them one at a time, and ``-lib-export-all`` offers every public function +of the entry module whose signature C can spell, naming the ones it skips. + +Every library gets the same four entry points, prefixed with the output +name — one instance owns one daslang context, with its own globals and +heap: + +.. code-block:: c + + #include "script.h" + + script_ctx * ctx = script_create(); + if ( !ctx ) { + printf("%s\n", script_last_error(NULL)); + return 1; + } + script_Vec3 v = { 1.0f, 2.0f, 3.0f }; + script_Vec3 out; + script_scale(ctx, &v, 2.0f, &out); /* a struct result uses a trailing out pointer */ + script_destroy(ctx); + +A daslang panic never unwinds into C: the call returns zero and leaves +``out`` untouched, and ``script_last_error(ctx)`` reports the text until +the next call on that instance clears it. A ``const char *`` a function +returns lives in that instance's string heap, so copy it if you need it +past the next call. + +The generated header asserts the layout of every structure it declares +(``_Static_assert`` in C11, ``static_assert`` in C++), so a host built +for a different target fails to compile rather than misreading memory. + +Several such libraries coexist in one process, and so does a library inside a +host that registered the daslang modules itself: whoever gets there first +registers the runtime, and the rest bind to it. Several instances of one +library are fine, on any thread. + + Serialization ============= diff --git a/doc/source/reference/language/annotations.rst b/doc/source/reference/language/annotations.rst index 15ec5828ff..31111aa93f 100644 --- a/doc/source/reference/language/annotations.rst +++ b/doc/source/reference/language/annotations.rst @@ -65,6 +65,40 @@ Lifecycle print("hello\n") } +``[export_c]`` + An ``[export]`` that also names the function to C: a standalone context declares it in the C + half of the header it generates, so a C host calls it with no daslang API of its own. Needs + ``require daslib/export_c``. The function keeps working in every tier. Its signature + has to be one C can spell: scalars, ``string``, raw pointers, enumerations, plain structures, + the ``float2``..``uint4`` and ``range`` families, and ``fixed_array`` arguments; ``array``, + ``table``, ``tuple``, ``variant``, lambdas, blocks, iterators and bound C++ types are not, and + the build names the parameter it could not spell. A generic function cannot carry it. + ``name="..."`` renames the C symbol, which is how two overloads reach C at all: + + .. das-doc: alt + .. code-block:: das + + struct Vec3 { + x : float + y : float + z : float + } + + [export_c] + def dot(a, b : Vec3) : float { + return a.x * b.x + a.y * b.y + a.z * b.z + } + + [export_c(name = "scale_by")] + def scale(v : Vec3; k : float) : Vec3 { + return Vec3(x = v.x * k, y = v.y * k, z = v.z * k) + } + + A scalar result comes back directly, so ``dot`` becomes + ``float p_dot(p_ctx *, const p_Vec3 *, const p_Vec3 *)``; a structure or vector result travels + through a trailing out pointer, so ``scale`` becomes + ``void p_scale_by(p_ctx *, const p_Vec3 *, float, p_Vec3 *)``. + ``[init]`` Marks a function to run automatically during context initialization. The function must take no arguments and return ``void``: diff --git a/doc/source/reference/tutorials/integration_cpp_20_standalone_contexts.rst b/doc/source/reference/tutorials/integration_cpp_20_standalone_contexts.rst index c6922674d6..2e2f38592e 100644 --- a/doc/source/reference/tutorials/integration_cpp_20_standalone_contexts.rst +++ b/doc/source/reference/tutorials/integration_cpp_20_standalone_contexts.rst @@ -113,7 +113,7 @@ no compilation, no simulation: TextPrinter tout; tout << "Creating standalone context...\n"; - auto ctx = standalone_context::Standalone(); + auto ctx = ctx_standalone_context::Standalone(); tout << "Calling test():\n"; ctx.test(); @@ -126,7 +126,7 @@ Key points: * **No** ``NEED_ALL_DEFAULT_MODULES`` or ``Module::Initialize`` — the standalone context is entirely self-contained. -* ``standalone_context::Standalone()`` — the constructor sets up all +* ``ctx_standalone_context::Standalone()`` — the constructor sets up all functions, globals, and type info from pre-generated AOT data. * ``ctx.test()`` — a direct C++ method call, not ``findFunction`` followed by ``evalWithCatch``. diff --git a/examples/standalone/01_pure/main.cpp b/examples/standalone/01_pure/main.cpp index e39e6e3e79..cdf1b5764b 100644 --- a/examples/standalone/01_pure/main.cpp +++ b/examples/standalone/01_pure/main.cpp @@ -37,18 +37,18 @@ int main () { // the context exists means even a panic during construction is visible. das_nano_set_print(&to_console); - pure_math::Standalone ctx; + ctx_pure_math::Standalone ctx; - pure_math::Vec3 a; a.x = 1.0f; a.y = 2.0f; a.z = 3.0f; - pure_math::Vec3 b; b.x = 4.0f; b.y = 5.0f; b.z = 6.0f; + ctx_pure_math::Vec3 a; a.x = 1.0f; a.y = 2.0f; a.z = 3.0f; + ctx_pure_math::Vec3 b; b.x = 4.0f; b.y = 5.0f; b.z = 6.0f; expect_float("dot(a,b)", ctx.dot(a, b), 32.0f); - pure_math::Vec3 s = ctx.scale(a, 2.0f); + ctx_pure_math::Vec3 s = ctx.scale(a, 2.0f); expect_float("scale(a,2).x", s.x, 2.0f); expect_float("scale(a,2).z", s.z, 6.0f); - expect_float("component(a,y)", ctx.component(a, pure_math::Axis::y), 2.0f); + expect_float("component(a,y)", ctx.component(a, ctx_pure_math::Axis::y), 2.0f); expect_float("weighted_sum(a)", ctx.weighted_sum(a), 1.0f*0.25f + 2.0f*0.5f + 3.0f*0.25f); expect_int("collatz_steps(27)", ctx.collatz_steps(27), 111); diff --git a/examples/standalone/02_heap/main.cpp b/examples/standalone/02_heap/main.cpp index 12821522a5..9b8cbba365 100644 --- a/examples/standalone/02_heap/main.cpp +++ b/examples/standalone/02_heap/main.cpp @@ -27,7 +27,7 @@ static void to_console ( const char * text ) { int main () { das_nano_set_print(&to_console); - heap_demo::Standalone ctx; + ctx_heap_demo::Standalone ctx; // 0..9 squared expect_int("sum_range(10)", ctx.sum_range(10), 285); diff --git a/examples/standalone/03_closures/main.cpp b/examples/standalone/03_closures/main.cpp index 0245079941..ae1bb2434c 100644 --- a/examples/standalone/03_closures/main.cpp +++ b/examples/standalone/03_closures/main.cpp @@ -26,7 +26,7 @@ static void to_console ( const char * text ) { int main () { das_nano_set_print(&to_console); - closures::Standalone ctx; + ctx_closures::Standalone ctx; expect_int("apply_twice(10)", ctx.apply_twice(10), 16); expect_int("call_through_pointer(21)", ctx.call_through_pointer(21), 42); diff --git a/examples/standalone/04_c_binding/main.cpp b/examples/standalone/04_c_binding/main.cpp index f4cf59e96e..c492124393 100644 --- a/examples/standalone/04_c_binding/main.cpp +++ b/examples/standalone/04_c_binding/main.cpp @@ -27,7 +27,7 @@ static void board_print ( const char * text ) { int main () { das_nano_set_print(&board_print); - blinker::Standalone ctx; + ctx_blinker::Standalone ctx; // One full sweep of the scanner: 1,2,4,8,4,2 then back to 1. static const unsigned expected[] = { 1, 2, 4, 8, 4, 2, 1, 2 }; diff --git a/examples/standalone/05_compile_time_table/main.cpp b/examples/standalone/05_compile_time_table/main.cpp index 265ca098aa..c0ea606c90 100644 --- a/examples/standalone/05_compile_time_table/main.cpp +++ b/examples/standalone/05_compile_time_table/main.cpp @@ -29,7 +29,7 @@ static int read_adc ( int sample ) { int main () { das_nano_set_print(&board_print); - thermometer::Standalone ctx; + ctx_thermometer::Standalone ctx; const int lo = ctx.adc_range_lo(); const int hi = ctx.adc_range_hi(); diff --git a/examples/standalone/06_full_runtime/main.cpp b/examples/standalone/06_full_runtime/main.cpp index 8ba4d0d6d2..9f9cd3754a 100644 --- a/examples/standalone/06_full_runtime/main.cpp +++ b/examples/standalone/06_full_runtime/main.cpp @@ -24,7 +24,7 @@ int main ( int argc, char * argv[] ) { return 7; } const char * url = argc > 1 ? argv[1] : "http://127.0.0.1:1/"; - service_probe::Standalone ctx; + ctx_service_probe::Standalone ctx; const int status = ctx.http_status((char *)url); printf("GET %s -> %d%s\n", url, status, status < 0 ? " (nobody answered)" : ""); const bool status_is_sane = status == -1 || (status >= 100 && status <= 599); diff --git a/examples/standalone/CMakeLists.txt b/examples/standalone/CMakeLists.txt index 256ad82b7d..dfcd8648f5 100644 --- a/examples/standalone/CMakeLists.txt +++ b/examples/standalone/CMakeLists.txt @@ -33,6 +33,7 @@ function(das_nano_example name dir das_file) ${ARGN} "${PROJECT_SOURCE_DIR}/utils/aot/main.das" "${PROJECT_SOURCE_DIR}/daslib/aot_standalone.das" + "${PROJECT_SOURCE_DIR}/daslib/c_api_header.das" "${PROJECT_SOURCE_DIR}/daslib/aot_cpp.das" WORKING_DIRECTORY "${PROJECT_SOURCE_DIR}" COMMENT "Standalone AOT: ${das_file}" @@ -69,6 +70,7 @@ if(NOT DAS_HV_DISABLED) "${CMAKE_CURRENT_SOURCE_DIR}/06_full_runtime/service_probe.das" "${PROJECT_SOURCE_DIR}/utils/aot/main.das" "${PROJECT_SOURCE_DIR}/daslib/aot_standalone.das" + "${PROJECT_SOURCE_DIR}/daslib/c_api_header.das" "${PROJECT_SOURCE_DIR}/daslib/aot_cpp.das" WORKING_DIRECTORY "${PROJECT_SOURCE_DIR}" COMMENT "Standalone AOT (full runtime): service_probe.das" diff --git a/skills/cpp_integration.md b/skills/cpp_integration.md index 6d12d75e9c..13b1d20bee 100644 --- a/skills/cpp_integration.md +++ b/skills/cpp_integration.md @@ -297,7 +297,7 @@ and link the result against nano instead: #include "script.das.h" das::das_nano_set_print(&my_uart_write); // every print leaves through this -script::Standalone ctx; // a plain C++ object +ctx_script::Standalone ctx; // a plain C++ object int answer = ctx.exported_function(21); ``` @@ -346,6 +346,56 @@ link. Worked example: `examples/standalone/06_full_runtime/` - read it for the s it needs the daslang repository, since the bundle carries no dasHV headers or archive. The recipe above works from a bundle for any C++ module you build yourself. +## Calling daslang from C - `daslang -lib` + +When the host is C, or wants no daslang API at all, compile the script to a native library with a +generated C header instead: + +```sh +daslang -lib script.das -output build/script # build/script.so (.dylib/.dll) + build/script.h +daslang -lib script.das -output build/script -- --jit-lib-static # build/script.a instead +``` + +Three ways to pick what crosses: mark each function `[export_c]` (an `[export]` the library also +surfaces in C); pass `-- --jit-lib-export-marked` to take whatever the program already marks +`[export]`; or pass `-lib-export-all` for every public function of the entry module whose signature +C can spell - only export-all skips an unspellable one with a warning, the other two make it a hard +error. `-lib` carries the annotation itself; add `require daslib/export_c` to compile that same +source without the JIT, which the linter and the AOT pass both do. `examples/c_api_library/` builds +one library each way and binds all three at once. + +A daslang host needs no C at all: `-- --jit-lib-bindings out/script_c.das` writes the daslang twin +of the header - one `[extern(cdecl, late, ...)]` per entry point plus a das struct per structure +that crosses - and the host `require`s that file. The externs are `late`, so the bindings compile +before the library exists and a build system can generate them as an ordinary output. + +For a C host, then: + +```c +#include "script.h" + +script_ctx * ctx = script_create(); /* one instance = one context, globals and heap */ +script_Vec3 v = { 1.0f, 2.0f, 3.0f }, out; +script_scale(ctx, &v, 2.0f, &out); /* a struct or vector result uses a trailing out ptr */ +if ( script_last_error(ctx) ) { /* the call raised; out is untouched */ } +script_destroy(ctx); +``` + +Scalars, `string` (as `const char *`), pointers and enums cross by value; everything else +representable crosses as `const T *`. A daslang panic returns zero and reports through +`script_last_error(ctx)` - it never unwinds into C. A returned `const char *` lives in that +instance's string heap, so copy it if you need it past the next call. The header asserts the +layout of every structure it declares, so a host built for a different target fails to compile. +Several such libraries coexist in one process, as does a library inside a host that registered +the daslang modules itself - the first one there registers the runtime and the rest bind to it. +Several instances of one library are fine, created and driven on any thread. + +Choosing between the four: **nano** when the host is C++ and you want the smallest runtime; a +**standalone context on the full runtime** when the host is C++ and the script reaches a C++ +module beyond `builtin`; **`-lib`** when the host is C, or wants a plain ABI boundary and no +daslang headers; the **C API** (`daScriptC.h`) when the host has to compile daslang itself at run +time. + ## Diagnostics - `TextPrinter`, never `fprintf(stderr, ...)` ```cpp diff --git a/skills/daslang/references/modules-and-stdlib.md b/skills/daslang/references/modules-and-stdlib.md index 3f3d768f12..9fc4a86305 100644 --- a/skills/daslang/references/modules-and-stdlib.md +++ b/skills/daslang/references/modules-and-stdlib.md @@ -22,7 +22,10 @@ def main { print("ok\n") } The only enforced ordering is `module` before any type declaration; `options` / `module` / `require` otherwise interleave. A file with no `module` line is a program, named by its file stem. -`[export]` makes a function callable from the host by name; `[init]` / `[finalize]` run at context +`[export]` makes a function callable from the host by name, and `[export_c]` is an `[export]` that +also crosses to C - a standalone context declares it in the C half of the header it generates, so a +C host needs no daslang API (`require daslib/export_c`; `[export_c(name = "...")]` picks the C +symbol, which is how two overloads both reach C); `[init]` / `[finalize]` run at context init / shutdown (no arguments, no return). `main` is a convention, not a keyword: it returns `void` unless declared `def main() : int`, whose return value is the process exit code (do not `panic` to force one). diff --git a/src/ast/ast_print.cpp b/src/ast/ast_print.cpp index dafe7966ec..98d06b2c1b 100644 --- a/src/ast/ast_print.cpp +++ b/src/ast/ast_print.cpp @@ -1471,7 +1471,7 @@ namespace das { void Program::setPrintFlags() { #if defined(STANDALONE_MODE) - ast_print::Standalone ctx; + ctx_ast_print::Standalone ctx; ctx.setFlags(this); #else ClearPrinterFlags cflags; diff --git a/src/builtin/module_builtin_ast_adapters.cpp b/src/builtin/module_builtin_ast_adapters.cpp index 42615a8a55..8239ebce5e 100644 --- a/src/builtin/module_builtin_ast_adapters.cpp +++ b/src/builtin/module_builtin_ast_adapters.cpp @@ -2541,6 +2541,17 @@ namespace das { program->visitModule(*adapter, module); } + void astVisitModuleWithSort ( smart_ptr_raw program, VisitorAdapter * adapter, + Module* module, bool sortStructures, Context * context, LineInfoArg * line_info ) { + if (!adapter) + context->throw_error_at(line_info, "adapter is required"); + if (!program) + context->throw_error_at(line_info, "program is required"); + if (!module) + context->throw_error_at(line_info, "module is required"); + program->visitModule(*adapter, module, false, sortStructures); + } + void astVisitModulesInOrder ( smart_ptr_raw program, VisitorAdapter * adapter, Context * context, LineInfoArg * line_info ) { if (!adapter) context->throw_error_at(line_info, "adapter is required"); @@ -2611,6 +2622,9 @@ namespace das { addExtern(*this, lib, "visit_modules", SideEffects::accessExternal, "astVisitModulesInOrder") ->args({"program","adapter","context","line"}); + addExtern(*this, lib, "visit_module", + SideEffects::accessExternal, "astVisitModuleWithSort") + ->args({"program","adapter","module","sortStructures","context","lineInfo"}); addExtern(*this, lib, "visit_module", SideEffects::accessExternal, "astVisitModule") ->args({"program","adapter","module","context","line"}); diff --git a/tests-cpp/big/nano_ctx/CMakeLists.txt b/tests-cpp/big/nano_ctx/CMakeLists.txt index 9cef9fba7c..6f8727cc54 100644 --- a/tests-cpp/big/nano_ctx/CMakeLists.txt +++ b/tests-cpp/big/nano_ctx/CMakeLists.txt @@ -44,6 +44,7 @@ foreach(_pair "01_pure/pure_math.das" "02_heap/heap_demo.das" "${PROJECT_SOURCE_DIR}/utils/aot/main.das" "${PROJECT_SOURCE_DIR}/daslib/aot_standalone.das" "${PROJECT_SOURCE_DIR}/daslib/aot_cpp.das" + "${PROJECT_SOURCE_DIR}/daslib/c_api_header.das" WORKING_DIRECTORY "${PROJECT_SOURCE_DIR}" COMMENT "Standalone AOT (nano): ${_das_name}" VERBATIM diff --git a/tests-cpp/big/nano_ctx/test_nano_ctx.cpp b/tests-cpp/big/nano_ctx/test_nano_ctx.cpp index 5593b33889..6b80b6238e 100644 --- a/tests-cpp/big/nano_ctx/test_nano_ctx.cpp +++ b/tests-cpp/big/nano_ctx/test_nano_ctx.cpp @@ -78,12 +78,12 @@ int main () { das_nano_set_print(&capture_print); { // tier A - POD compute, no das heap - pure_math::Standalone ctx; - pure_math::Vec3 a; a.x = 1.0f; a.y = 2.0f; a.z = 3.0f; - pure_math::Vec3 b; b.x = 4.0f; b.y = 5.0f; b.z = 6.0f; + ctx_pure_math::Standalone ctx; + ctx_pure_math::Vec3 a; a.x = 1.0f; a.y = 2.0f; a.z = 3.0f; + ctx_pure_math::Vec3 b; b.x = 4.0f; b.y = 5.0f; b.z = 6.0f; expect_float("dot", ctx.dot(a, b), 32.0f); expect_float("scale.y", ctx.scale(a, 3.0f).y, 6.0f); - expect_float("component(z)", ctx.component(a, pure_math::Axis::z), 3.0f); + expect_float("component(z)", ctx.component(a, ctx_pure_math::Axis::z), 3.0f); expect_float("weighted_sum", ctx.weighted_sum(a), 2.0f); expect_int("collatz_steps(27)", ctx.collatz_steps(27), 111); // `options stack = 4096` is honored exactly, plus the headroom the @@ -103,7 +103,7 @@ int main () { } { // tier B - the das heap - heap_demo::Standalone ctx; + ctx_heap_demo::Standalone ctx; expect_int("sum_range(10)", ctx.sum_range(10), 285); expect_int("histogram_peak(20)", ctx.histogram_peak(20), 3); expect_int("alloc_and_free(8)", int(ctx.alloc_and_free(8)), 4); @@ -112,7 +112,7 @@ int main () { } { // tier C - lambdas, function pointers, generators - closures::Standalone ctx; + ctx_closures::Standalone ctx; expect_int("apply_twice(10)", ctx.apply_twice(10), 16); expect_int("call_through_pointer(21)", ctx.call_through_pointer(21), 42); expect_int("sum_squares(5)", ctx.sum_squares(5), 30); @@ -122,7 +122,7 @@ int main () { } { // output - `print` reaches the embedder's sink and nowhere else - blinker::Standalone ctx; + ctx_blinker::Standalone ctx; expect_int("lamp_pattern(3)", ctx.lamp_pattern(3), 8); expect_int("lamp_pattern(4)", ctx.lamp_pattern(4), 4); g_captured.clear(); @@ -175,7 +175,7 @@ int main () { // that the numbers survived into the linked program at all: nothing in this // binary can read thermistor.csv, so a wrong table cannot be recovered at // run time - it can only be wrong. - thermometer::Standalone ctx; + ctx_thermometer::Standalone ctx; expect_int("baked table size", ctx.curve_size(), 64); expect_int("baked ADC low", ctx.adc_range_lo(), 267); expect_int("baked ADC high", ctx.adc_range_hi(), 3740); diff --git a/tests-cpp/big/standalone_ctx/CMakeLists.txt b/tests-cpp/big/standalone_ctx/CMakeLists.txt index 4879c2f353..d36abaaeea 100644 --- a/tests-cpp/big/standalone_ctx/CMakeLists.txt +++ b/tests-cpp/big/standalone_ctx/CMakeLists.txt @@ -7,16 +7,22 @@ file(MAKE_DIRECTORY "${STANDALONE_CTX_GEN}") add_custom_command( OUTPUT "${STANDALONE_CTX_GEN}/standalone_init_fixture.das.cpp" "${STANDALONE_CTX_GEN}/standalone_init_fixture.das.h" + "${CMAKE_CURRENT_SOURCE_DIR}/_standalone_init_fixture_c.das" COMMAND $ "${PROJECT_SOURCE_DIR}/utils/aot/main.das" -- -ctx "${CMAKE_CURRENT_SOURCE_DIR}/standalone_init_fixture.das" "${STANDALONE_CTX_GEN}/" + # the bindings live beside the host that requires them; the library they name is the one + # the shared target below builds out of the same generated .cpp + -bindings "${CMAKE_CURRENT_SOURCE_DIR}/_standalone_init_fixture_c.das" + -bindings_library "${CMAKE_CURRENT_BINARY_DIR}/standalone_init_fixture" DEPENDS daslang "${CMAKE_CURRENT_SOURCE_DIR}/standalone_init_fixture.das" "${CMAKE_CURRENT_SOURCE_DIR}/standalone_init_dep.das" "${PROJECT_SOURCE_DIR}/utils/aot/main.das" "${PROJECT_SOURCE_DIR}/daslib/aot_standalone.das" "${PROJECT_SOURCE_DIR}/daslib/aot_cpp.das" + "${PROJECT_SOURCE_DIR}/daslib/c_api_header.das" WORKING_DIRECTORY "${PROJECT_SOURCE_DIR}" COMMENT "Standalone AOT: standalone_init_fixture.das" VERBATIM @@ -36,6 +42,100 @@ add_test(NAME standalone_ctx COMMAND test_standalone_ctx set_tests_properties(standalone_ctx PROPERTIES LABELS "big") add_dependencies(test-big test_standalone_ctx) +# The same fixture driven through its C API, with the C half of the generated header compiled by +# a C compiler: a header that is not valid C, a layout that disagrees with das, or a thunk that +# loses a value fails here. The C++ twin above shares the generated source, so both APIs are +# proven against one emission. +add_executable(test_standalone_capi + test_standalone_capi.c + "${STANDALONE_CTX_GEN}/standalone_init_fixture.das.cpp") +target_link_libraries(test_standalone_capi PRIVATE + libDaScript ${SRC_LIBRARIES} ${DAS_MODULES_LIBS}) +target_include_directories(test_standalone_capi PRIVATE "${STANDALONE_CTX_GEN}" ${NEED_MODULES_PATH}) +set_target_properties(test_standalone_capi PROPERTIES + FOLDER "tests-cpp/big" + C_STANDARD 11 + C_STANDARD_REQUIRED ON + LINKER_LANGUAGE CXX) +SETUP_CPP11(test_standalone_capi) + +add_test(NAME standalone_capi COMMAND test_standalone_capi + WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}) +set_tests_properties(standalone_capi PROPERTIES LABELS "small") +add_dependencies(test-small test_standalone_capi) + +# The same emission, loaded instead of linked: a shared library built from the generated .cpp +# exports the C entry points (default visibility; on Windows the header's _API macro is +# what says so), and a daslang host reaches them through the bindings the same run wrote. Needs +# the shared runtime, since the host process already carries one. +if(TARGET libDaScriptDyn_runtime) + add_library(standalone_init_fixture_shared SHARED + "${STANDALONE_CTX_GEN}/standalone_init_fixture.das.cpp") + target_link_libraries(standalone_init_fixture_shared PRIVATE + libDaScriptDyn_runtime ${SRC_LIBRARIES} ${DAS_MODULES_LIBS}) + target_include_directories(standalone_init_fixture_shared PRIVATE + "${STANDALONE_CTX_GEN}" ${NEED_MODULES_PATH}) + target_compile_definitions(standalone_init_fixture_shared PRIVATE + STANDALONE_INIT_FIXTURE_SHARED STANDALONE_INIT_FIXTURE_BUILD) + # the shared runtime hides its typeinfo (SETUP_DAS_SHARED_LIBRARY, CXX_VISIBILITY_PRESET + # hidden), so a library carrying `class Standalone : public Context` cannot resolve + # typeinfo for das::Context at load. No RTTI, no reference - the same rule a shared module obeys. + target_compile_options(standalone_init_fixture_shared PRIVATE + $<$:-fno-rtti> + $<$:/GR->) + set_target_properties(standalone_init_fixture_shared PROPERTIES + PREFIX "" + OUTPUT_NAME standalone_init_fixture + LIBRARY_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}" + RUNTIME_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}" + BUILD_RPATH "${PROJECT_SOURCE_DIR}/lib" + FOLDER "tests-cpp/big") + SETUP_CPP11(standalone_init_fixture_shared) + + add_test(NAME standalone_capi_dasbind + COMMAND $ + "${CMAKE_CURRENT_SOURCE_DIR}/test_standalone_bindings_host.das" + WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}) + set_tests_properties(standalone_capi_dasbind PROPERTIES LABELS "small") + add_dependencies(test-small standalone_init_fixture_shared) +endif() + +# The generated header asserts every structure's size and each of its field offsets. Nothing +# proved those asserts FIRE, which is the whole reason they are emitted, so this compiles a host +# that packs its structures and requires the compiler to refuse it. Skipped on MSVC: the C flags +# and the _Static_assert spelling both differ there. +if(NOT MSVC) + add_custom_command( + OUTPUT "${STANDALONE_CTX_GEN}/standalone_layout_fixture.das.h" + "${STANDALONE_CTX_GEN}/standalone_layout_fixture.das.cpp" + COMMAND $ + "${PROJECT_SOURCE_DIR}/utils/aot/main.das" + -- -ctx "${CMAKE_CURRENT_SOURCE_DIR}/standalone_layout_fixture.das" + "${STANDALONE_CTX_GEN}/" + DEPENDS daslang + "${CMAKE_CURRENT_SOURCE_DIR}/standalone_layout_fixture.das" + "${PROJECT_SOURCE_DIR}/utils/aot/main.das" + "${PROJECT_SOURCE_DIR}/daslib/aot_standalone.das" + "${PROJECT_SOURCE_DIR}/daslib/aot_cpp.das" + "${PROJECT_SOURCE_DIR}/daslib/c_api_header.das" + WORKING_DIRECTORY "${PROJECT_SOURCE_DIR}" + COMMENT "Standalone AOT: standalone_layout_fixture.das" + VERBATIM + ) + add_custom_target(standalone_layout_fixture_header + DEPENDS "${STANDALONE_CTX_GEN}/standalone_layout_fixture.das.h") + set_target_properties(standalone_layout_fixture_header PROPERTIES FOLDER "tests-cpp/big") + + add_test(NAME standalone_layout_assert COMMAND ${CMAKE_COMMAND} + -DCC=${CMAKE_C_COMPILER} + -DSRC=${CMAKE_CURRENT_SOURCE_DIR}/test_standalone_layout_packed.c + -DINC=${STANDALONE_CTX_GEN} + -DOUT=${CMAKE_CURRENT_BINARY_DIR}/packed_host.o + -P ${CMAKE_CURRENT_SOURCE_DIR}/expect_layout_assert.cmake) + set_tests_properties(standalone_layout_assert PROPERTIES LABELS "small") + add_dependencies(test-small standalone_layout_fixture_header) +endif() + # Two contexts with different C++ module sets in one binary - the dasHV + fio example and a # fio-only fixture - share one module registry lifetime. Small-labelled: a second owner of the # shutdown, or an unbalanced Initialize, fails at exit and nothing else would notice. @@ -54,6 +154,7 @@ if(NOT DAS_HV_DISABLED) "${PROJECT_SOURCE_DIR}/utils/aot/main.das" "${PROJECT_SOURCE_DIR}/daslib/aot_standalone.das" "${PROJECT_SOURCE_DIR}/daslib/aot_cpp.das" + "${PROJECT_SOURCE_DIR}/daslib/c_api_header.das" WORKING_DIRECTORY "${PROJECT_SOURCE_DIR}" COMMENT "Standalone AOT: ${_das_name}" VERBATIM diff --git a/tests-cpp/big/standalone_ctx/expect_layout_assert.cmake b/tests-cpp/big/standalone_ctx/expect_layout_assert.cmake new file mode 100644 index 0000000000..222d16a98f --- /dev/null +++ b/tests-cpp/big/standalone_ctx/expect_layout_assert.cmake @@ -0,0 +1,29 @@ +# Proves the generated header's layout guard fires. The packed host must fail to compile, and it +# must fail ON a layout assert - any other error would let a broken guard pass this test. +# +# cmake -DCC= -DSRC= -DINC= -DOUT= -P this + +foreach(_var CC SRC INC OUT) + if(NOT DEFINED ${_var}) + message(FATAL_ERROR "expect_layout_assert.cmake: -D${_var} is required") + endif() +endforeach() + +execute_process( + COMMAND ${CC} -std=c11 -I${INC} -c ${SRC} -o ${OUT} + RESULT_VARIABLE _rc + OUTPUT_VARIABLE _log + ERROR_VARIABLE _log +) + +if(_rc EQUAL 0) + message(FATAL_ERROR + "the packed host compiled: the generated header's layout asserts did not fire") +endif() + +string(FIND "${_log}" "differs from the daslang layout" _found) +if(_found EQUAL -1) + message(FATAL_ERROR "the packed host failed, but not on a layout assert:\n${_log}") +endif() + +message(STATUS "the layout assert fired, as it must") diff --git a/tests-cpp/big/standalone_ctx/standalone_init_fixture.das b/tests-cpp/big/standalone_ctx/standalone_init_fixture.das index 75ed20f363..af4e807d0f 100644 --- a/tests-cpp/big/standalone_ctx/standalone_init_fixture.das +++ b/tests-cpp/big/standalone_ctx/standalone_init_fixture.das @@ -2,6 +2,7 @@ options gen2 options stack = 262144 require standalone_init_dep +require daslib/export_c var g_stamp = 0 @@ -85,6 +86,32 @@ def flip(m : Mode) : Mode { return m == Mode.on ? Mode.off : Mode.on } +struct Outer { + inner : Inner + tag : int +} + +struct Inner { + weight : int +} + +[export] +def outer_weight(o : Outer) : int { + return o.inner.weight + o.tag +} + +struct Node { + value : int + next : Node? +} + +var private g_node : Node? + +[export] +def head_value : int { + return g_node?.value ?? 41 +} + [export] def apply_lambda(x : int) : int { let add3 = @(v : int) : int => v + 3 @@ -115,3 +142,8 @@ def sum_generator(n : int) : int { } return s } + +[export_c(name = "renamed_sum")] +def sum_under_another_name(a, b : int) : int { + return a + b +} diff --git a/tests-cpp/big/standalone_ctx/standalone_layout_fixture.das b/tests-cpp/big/standalone_ctx/standalone_layout_fixture.das new file mode 100644 index 0000000000..654aab2a9d --- /dev/null +++ b/tests-cpp/big/standalone_ctx/standalone_layout_fixture.das @@ -0,0 +1,15 @@ +options gen2 + + +require daslib/export_c + +struct Wide { + small : int8 + wide : int64 + fraction : float +} + +[export_c] +def widen(n : int) : Wide { + return Wide(small = int8(n), wide = int64(n), fraction = float(n)) +} diff --git a/tests-cpp/big/standalone_ctx/test_standalone_bindings_host.das b/tests-cpp/big/standalone_ctx/test_standalone_bindings_host.das new file mode 100644 index 0000000000..9e340efd31 --- /dev/null +++ b/tests-cpp/big/standalone_ctx/test_standalone_bindings_host.das @@ -0,0 +1,30 @@ +options gen2 + +require daslib/safe_addr +require _standalone_init_fixture_c + +[export] +def main() : int { + var ctx = standalone_init_fixture_create() + if (ctx == null) { + print("create failed: {standalone_init_fixture_last_error(null)}\n") + return 1 + } + var bad = 0 + let first = standalone_init_fixture_get_first(ctx) + let stamp = standalone_init_fixture_get_init_fn_stamp(ctx) + let shared_total = standalone_init_fixture_get_shared_total(ctx) + if (first != 31 || stamp != 3 || shared_total != 6) { + print("globals answered first={first} stamp={stamp} shared={shared_total}\n") + bad++ + } + var p = standalone_init_fixture_Pair(a = 40, b = 2) + let pair_sum = standalone_init_fixture_pair_sum(ctx, safe_addr(p)) + let renamed = standalone_init_fixture_renamed_sum(ctx, 20, 22) + if (pair_sum != 42 || renamed != 42) { + print("calls answered pair_sum={pair_sum} renamed={renamed}\n") + bad++ + } + standalone_init_fixture_destroy(ctx) + return bad +} diff --git a/tests-cpp/big/standalone_ctx/test_standalone_capi.c b/tests-cpp/big/standalone_ctx/test_standalone_capi.c new file mode 100644 index 0000000000..e7854c0cd6 --- /dev/null +++ b/tests-cpp/big/standalone_ctx/test_standalone_capi.c @@ -0,0 +1,65 @@ +#include "standalone_init_fixture.das.h" + +#include +#include + +static int failures = 0; + +static void expect ( const char * what, int have, int want ) { + if ( have != want ) { + printf("%s = %d, expected %d\n", what, have, want); + failures ++; + } +} + +int main ( void ) { + standalone_init_fixture_ctx * ctx = standalone_init_fixture_create(); + if ( !ctx ) { + printf("create failed: %s\n", standalone_init_fixture_last_error(NULL)); + return 1; + } + + expect("get_first()", standalone_init_fixture_get_first(ctx), 31); + expect("get_second()", standalone_init_fixture_get_second(ctx), 2); + expect("get_init_fn_stamp()", standalone_init_fixture_get_init_fn_stamp(ctx), 3); + expect("get_reads_forward()", standalone_init_fixture_get_reads_forward(ctx), 100); + expect("get_later()", standalone_init_fixture_get_later(ctx), 7); + expect("get_shared_total()", standalone_init_fixture_get_shared_total(ctx), 6); + + standalone_init_fixture_Pair made; + memset(&made, 0xAA, sizeof(made)); + standalone_init_fixture_make_pair(ctx, 3, 4, &made); + expect("make_pair(3,4).a", made.a, 3); + expect("make_pair(3,4).b", made.b, 4); + expect("pair_sum(make_pair(3,4))", standalone_init_fixture_pair_sum(ctx, &made), 7); + + standalone_init_fixture_Pair mine; + mine.a = 40; + mine.b = 2; + expect("pair_sum(a Pair this C host built)", standalone_init_fixture_pair_sum(ctx, &mine), 42); + + expect("flip(on)", standalone_init_fixture_flip(ctx, standalone_init_fixture_Mode_on), + standalone_init_fixture_Mode_off); + expect("flip(off)", standalone_init_fixture_flip(ctx, standalone_init_fixture_Mode_off), + standalone_init_fixture_Mode_on); + + standalone_init_fixture_Outer nested; + nested.inner.weight = 9; + nested.tag = 5; + expect("outer_weight(a struct holding a struct)", + standalone_init_fixture_outer_weight(ctx, &nested), 14); + + expect("head_value() on a null safe-navigation", standalone_init_fixture_head_value(ctx), 41); + + expect("renamed_sum(20,22) through the [export_c(name=...)] symbol", + standalone_init_fixture_renamed_sum(ctx, 20, 22), 42); + + if ( standalone_init_fixture_last_error(ctx) ) { + printf("a call raised: %s\n", standalone_init_fixture_last_error(ctx)); + failures ++; + } + + standalone_init_fixture_destroy(ctx); + printf(failures ? "standalone_capi: %d failure(s)\n" : "standalone_capi: ok\n", failures); + return failures ? 1 : 0; +} diff --git a/tests-cpp/big/standalone_ctx/test_standalone_ctx.cpp b/tests-cpp/big/standalone_ctx/test_standalone_ctx.cpp index d406ead92b..6079fb8f4c 100644 --- a/tests-cpp/big/standalone_ctx/test_standalone_ctx.cpp +++ b/tests-cpp/big/standalone_ctx/test_standalone_ctx.cpp @@ -4,7 +4,7 @@ using namespace das; int main( int, char * [] ) { - standalone_init_fixture::Standalone ctx; + ctx_standalone_init_fixture::Standalone ctx; TextPrinter tout; int failures = 0; auto expect = [&]( const char * name, int32_t have, int32_t want ) { @@ -18,16 +18,16 @@ int main( int, char * [] ) { expect("get_init_fn_stamp()", ctx.get_init_fn_stamp(), 3); expect("get_reads_forward()", ctx.get_reads_forward(), 100); expect("get_later()", ctx.get_later(), 7); - standalone_init_fixture::Pair madePair = ctx.make_pair(3, 4); + ctx_standalone_init_fixture::Pair madePair = ctx.make_pair(3, 4); expect("make_pair(3,4).a", madePair.a, 3); expect("make_pair(3,4).b", madePair.b, 4); expect("pair_sum(make_pair(3,4))", ctx.pair_sum(madePair), 7); - standalone_init_fixture::Pair embedderPair; + ctx_standalone_init_fixture::Pair embedderPair; embedderPair.a = 20; embedderPair.b = 22; expect("pair_sum(embedder-built Pair)", ctx.pair_sum(embedderPair), 42); - expect("flip(on)", int32_t(ctx.flip(standalone_init_fixture::Mode::on)), - int32_t(standalone_init_fixture::Mode::off)); + expect("flip(on)", int32_t(ctx.flip(ctx_standalone_init_fixture::Mode::on)), + int32_t(ctx_standalone_init_fixture::Mode::off)); constexpr uint32_t kFixtureStackSize = 262144; expect("stack.size() exceeds options stack by the init headroom", ctx.stack.size() > kFixtureStackSize ? 1 : 0, 1); @@ -48,6 +48,7 @@ int main( int, char * [] ) { expect("findVariable(nope)", ctx.findVariable("nope"), -1); // a shared global sits in the shared block: its emitted offset is the shared running size expect("get_shared_total()", ctx.get_shared_total(), 6); + expect("renamed_sum(20,22)", ctx.renamed_sum(20, 22), 42); int sharedTaps = ctx.findVariable("g_shared_taps"); expect("findVariable(g_shared_taps)", sharedTaps >= 0 ? 1 : 0, 1); expect("getVariable(g_shared_taps)[0]", sharedTaps >= 0 ? ((int32_t *) ctx.getVariable(sharedTaps))[0] : -1, 1); diff --git a/tests-cpp/big/standalone_ctx/test_standalone_layout_packed.c b/tests-cpp/big/standalone_ctx/test_standalone_layout_packed.c new file mode 100644 index 0000000000..abace90abf --- /dev/null +++ b/tests-cpp/big/standalone_ctx/test_standalone_layout_packed.c @@ -0,0 +1,8 @@ + +#pragma pack(push, 1) +#include "standalone_layout_fixture.das.h" +#pragma pack(pop) + +int main ( void ) { + return 0; +} diff --git a/tests-cpp/big/standalone_ctx/test_standalone_modules.cpp b/tests-cpp/big/standalone_ctx/test_standalone_modules.cpp index a55cc64b3b..2599d8ec40 100644 --- a/tests-cpp/big/standalone_ctx/test_standalone_modules.cpp +++ b/tests-cpp/big/standalone_ctx/test_standalone_modules.cpp @@ -26,12 +26,12 @@ DECLARE_MODULE(Module_HV); static int run_contexts ( char * self ) { int failures = 0; - standalone_modules_fixture::Standalone fio_only; + ctx_standalone_modules_fixture::Standalone fio_only; if ( !fio_only.has_path_variable() ) { printf("has_path_variable() = false, expected true\n"); failures ++; } - service_probe::Standalone probe; + ctx_service_probe::Standalone probe; const int status = probe.http_status((char *)"http://127.0.0.1:1/"); if ( status != -1 ) { printf("http_status(dead port) = %d, expected -1\n", status); diff --git a/tests/aot/test_standalone_emit.das b/tests/aot/test_standalone_emit.das index e33a5e0b4c..2d5c4e59d7 100644 --- a/tests/aot/test_standalone_emit.das +++ b/tests/aot/test_standalone_emit.das @@ -161,7 +161,6 @@ def test_standalone_emit(t : T?) { // nolint:STYLE038 - flat list of emit subc t |> success(a_pos < b_pos && b_pos < main_pos, "dependency-first order: dep_a, dep_b, entry module") t |> success(find(files.header, "static_assert(sizeof(DepSpan)") >= 0, "a required module's struct definition is in the header") t |> success(find(files.header, "struct _standalone_dep_a::DepSpan") >= 0, "the method spells the required module's struct by its public name") - t |> success(find(files.source, "static_assert(sizeof(DepSpan)") < 0, "the source does not redefine the required module's struct") } t |> run("a required module's used functions emit into the one translation unit, and link their externs' modules") @(t : T?) { @@ -194,23 +193,26 @@ def test_standalone_emit(t : T?) { // nolint:STYLE038 - flat list of emit subc let files = generate_standalone_files(t, "_standalone_struct_sig_fixture", out_dir) t |> success(find(files.header, "static_assert(sizeof(Pair)") >= 0, "the struct definition is in the header") t |> success(find(files.header, "enum class Mode") >= 0, "the enum definition is in the header") - t |> success(find(files.header, "-> struct _standalone_struct_sig_fixture::Pair") >= 0, "the method spells the struct by its public name") - t |> success(find(files.header, "DAS_COMMENT(enum) _standalone_struct_sig_fixture::Mode") >= 0, "the method spells the enum by its public name") + t |> success(find(files.header, "-> struct ctx__standalone_struct_sig_fixture::Pair") >= 0, "the method spells the struct by its public name") + t |> success(find(files.header, "DAS_COMMENT(enum) ctx__standalone_struct_sig_fixture::Mode") >= 0, "the method spells the enum by its public name") t |> success(find(files.header, "pair_sum") >= 0, "the struct-taking method is declared") t |> success(find(files.header, "namespace \{") < 0, "no 'namespace \{' (empty module name) in the header") t |> success(find(files.header, "#pragma once") >= 0, "the header guards against double inclusion") t |> success(find(files.header, "bin_serializer.h") >= 0, "required-module includes are in the header") - t |> success(find(files.source, "static_assert(sizeof(Pair)") < 0, "the source does not redefine the struct") - t |> success(find(files.source, "enum class Mode") < 0, "the source does not redefine the enum") + t |> success(find(files.source, "class Standalone") >= 0 && find(files.source, "static_assert(sizeof(Pair)") >= 0, + "the .cpp carries the header's text instead of including it, so one copy defines the types") + t |> success(find(files.source, "#pragma once") < 0, "no #pragma once in the .cpp - it is a main file") + t |> success(find(files.source, "#include \"_standalone_struct_sig_fixture.das.h\"") < 0, + "the .cpp does not include the header it inlined") t |> equal(brace_balance(files.header), 0, "unbalanced braces in the generated header") t |> equal(brace_balance(files.source), 0, "unbalanced braces in the generated C++") } t |> run("cross-platform debug info spells entry-module types by their public name") @(t : T?) { let files = generate_standalone_files(t, "_standalone_struct_sig_fixture", out_dir, true) - t |> success(find(files.source, "offsetof(_standalone_struct_sig_fixture::Pair,") >= 0, "offsetof names the qualified struct") - t |> success(find(files.source, "TypeSize<_standalone_struct_sig_fixture::Pair>") >= 0, "TypeSize names the qualified struct") - t |> success(find(files.source, "DAS_COMMENT(enum) _standalone_struct_sig_fixture::Mode>") >= 0, "TypeSize names the qualified enum") + t |> success(find(files.source, "offsetof(ctx__standalone_struct_sig_fixture::Pair,") >= 0, "offsetof names the qualified struct") + t |> success(find(files.source, "TypeSize") >= 0, "TypeSize names the qualified struct") + t |> success(find(files.source, "DAS_COMMENT(enum) ctx__standalone_struct_sig_fixture::Mode>") >= 0, "TypeSize names the qualified enum") t |> success(find(files.source, "offsetof(Pair,") < 0, "no unqualified offsetof survives") t |> success(find(files.source, "TypeSize") < 0, "no unqualified struct TypeSize survives") t |> equal(brace_balance(files.source), 0, "unbalanced braces in the generated C++") @@ -222,7 +224,7 @@ def test_standalone_emit(t : T?) { // nolint:STYLE038 - flat list of emit subc // [no_aot] check below then reported its visit() as an unemittable function. let source = generate_standalone_source(t, "_standalone_macro_fixture", out_dir) t |> success(find(source, "5050") >= 0, "the macro's compile-time sum is baked into the source") - t |> success(find(source, "BakeSum") < 0, "the macro class does not reach the generated source") + t |> success(find(source, "struct BakeSum") < 0, "the macro class is not emitted (a comment naming it is fine)") t |> equal(brace_balance(source), 0, "unbalanced braces in the generated C++") } diff --git a/tutorials/integration/cpp/20_standalone_context.cpp b/tutorials/integration/cpp/20_standalone_context.cpp index 5b18678552..bca5097571 100644 --- a/tutorials/integration/cpp/20_standalone_context.cpp +++ b/tutorials/integration/cpp/20_standalone_context.cpp @@ -40,7 +40,7 @@ int main(int, char * []) { // Instantiate the standalone context. // The constructor sets up all functions, globals, and type info // from pre-generated AOT data. - auto ctx = standalone_context::Standalone(); + auto ctx = ctx_standalone_context::Standalone(); // Call the test function directly — this is a normal C++ method call, // not a lookup + eval. Maximum performance, minimum overhead. diff --git a/utils/CMakeLists.txt b/utils/CMakeLists.txt index e4b58bc4fe..7608ffe862 100644 --- a/utils/CMakeLists.txt +++ b/utils/CMakeLists.txt @@ -37,6 +37,7 @@ if(TARGET libDasModuleHV AND TARGET libDasModuleStdDlg AND TARGET libDasModuleSt "${PROJECT_SOURCE_DIR}/utils/aot/main.das" "${PROJECT_SOURCE_DIR}/daslib/aot_standalone.das" "${PROJECT_SOURCE_DIR}/daslib/aot_cpp.das" + "${PROJECT_SOURCE_DIR}/daslib/c_api_header.das" WORKING_DIRECTORY "${PROJECT_SOURCE_DIR}" COMMENT "Standalone AOT (full runtime): watchdog" VERBATIM diff --git a/utils/aot/main.das b/utils/aot/main.das index 4226f6a0ad..bebaed3052 100644 --- a/utils/aot/main.das +++ b/utils/aot/main.das @@ -189,9 +189,15 @@ def main() { } } updateCOP(cop, !gen1, gen2_make, false) + let bindings = find_argument_or(args, "-bindings", "") + let bindings_library = find_argument_or(args, "-bindings_library", "") + if (!bindings.empty() && length(ctx_files) != 1) { + panic("-bindings names one output, so it takes exactly one -ctx input") + } for ((ctx_in, ctx_out) in ctx_files) { ast_gc_guard() { - let is_ok = standalone_aot(ctx_in, ctx_out, cross_platform, false, cop) + let is_ok = standalone_aot(ctx_in, ctx_out, bindings, bindings_library, + cross_platform, false, cop) if (!is_ok && !quiet) { to_log(LOG_ERROR, "Failed to compile `{ctx_in}` in standalone.\n") } diff --git a/utils/watchdog/main.cpp b/utils/watchdog/main.cpp index 05ba1bcd1f..41713b7b8d 100644 --- a/utils/watchdog/main.cpp +++ b/utils/watchdog/main.cpp @@ -34,7 +34,7 @@ int main ( int argc, char * argv[] ) { #else const int32_t pid = (int32_t)getpid(); #endif - main::Standalone ctx; + ctx_main::Standalone ctx; if ( !ctx.start(pid) ) return ctx.result(); bool done = false; while ( !done ) { From d700898d646db73062573543c668cbfdd61471fd Mon Sep 17 00:00:00 2001 From: Churkin Aleksey Date: Wed, 9 Sep 2026 02:00:58 +0300 Subject: [PATCH 2/3] jit: daslang -lib emits a C-ABI native library The JIT could produce an exe and an AOT object; a host that only calls one script had neither. -lib writes a shared library (or an archive under --jit-lib-static) plus a C header, and links no daslang API into the host. The C surface is daslib/c_api_header's, so a JIT library and a standalone context declare the same API and skip the same signatures. Which functions cross is Function.flags.exports, read three ways: [export_c] alone by default, the bit however it was set under --jit-lib-export-marked, and every public function under -lib-export-all, which is a policy because MarkSymbolUse consumes it before infer. Codegen stays in LlvmJitMode.EXE, so inject_main's startup is shared. Each export gets an extern "C" thunk that packs its arguments into a frame and hands it to a trampoline through jit_lib_invoke_guarded, so a das panic reaches the caller as a zero return with its text in

_last_error, never as an unwind through C. Nothing in the emit path panics: the artifact writers return whether they wrote and run_jit collects it. A library initializes the modules it registers - the host's own runtime is already initialized, but an uninitialized module carries no functions, which an extern lookup reported as a missing function in a module that plainly existed. Two opt-in sweeps drive the corpus test_aot covers, one per backend: the C++ tier compiles and launches every context in one process, the JIT tier emits each library and loads it back. ARCHITECTURE_LIB.md#lib-runtime-scope records what a library's runtime guarantees. --- .gitignore | 7 + CLAUDE.md | 2 +- CMakeLists.txt | 3 + daslib/ARCHITECTURE_CAPI.md | 76 ++-- doc/source/reference/embedding/advanced.rst | 9 +- doc/source/reference/embedding/c_api.rst | 6 + doc/source/reference/language/annotations.rst | 5 +- ...ructure_annotation-rtti-CodeOfPolicies.rst | 1 + examples/c_api_library/CMakeLists.txt | 57 +++ examples/c_api_library/greetings.das | 17 + examples/c_api_library/main.das | 43 ++ examples/c_api_library/shapes.das | 24 ++ examples/c_api_library/units.das | 17 + include/daScript/ast/dyn_modules.h | 1 + include/daScript/simulate/code_of_policies.h | 1 + install/CLAUDE.md | 2 +- modules/dasLLVM/.das_module | 2 +- modules/dasLLVM/ARCHITECTURE.md | 27 +- modules/dasLLVM/ARCHITECTURE_LIB.md | 67 +++ modules/dasLLVM/README.md | 66 +++ modules/dasLLVM/REVIEW.md | 8 + modules/dasLLVM/daslib/jit_standalone.das | 327 +++++++++++++++ modules/dasLLVM/daslib/llvm_dll_utils.das | 34 +- modules/dasLLVM/daslib/llvm_exe.das | 390 ++++++++++++++---- modules/dasLLVM/daslib/llvm_jit.das | 4 +- modules/dasLLVM/daslib/llvm_jit_cli.das | 16 + modules/dasLLVM/daslib/llvm_jit_common.das | 80 ++-- modules/dasLLVM/daslib/llvm_jit_link.das | 2 +- modules/dasLLVM/daslib/llvm_jit_plan.das | 18 +- modules/dasLLVM/daslib/llvm_jit_run.das | 57 ++- skills/daslang/references/cli-and-config.md | 3 + .../daslang/references/modules-and-stdlib.md | 6 +- src/ast/ast_export.cpp | 9 + src/ast/ast_module.cpp | 3 - src/ast/ast_parse.cpp | 3 - src/builtin/jit_runtime.cpp | 77 +++- src/builtin/module_builtin_rtti.cpp | 1 + src/builtin/module_jit.cpp | 26 ++ tests-cpp/big/standalone_ctx/CMakeLists.txt | 10 +- tests-cpp/small/test_jit_lib_guard.cpp | 114 +++++ tests/CMakeLists.txt | 3 + tests/jit_tests/_jit_lib_align.das | 15 + tests/jit_tests/_jit_lib_align_host.das | 22 + tests/jit_tests/_jit_lib_bind.das | 35 ++ tests/jit_tests/_jit_lib_bind_host.das | 47 +++ tests/jit_tests/_jit_lib_export_all_host.das | 20 + tests/jit_tests/_jit_lib_guest.das | 16 + tests/jit_tests/_jit_lib_guest_host.das | 20 + tests/jit_tests/_jit_lib_probe.das | 32 ++ tests/jit_tests/_jit_lib_probe_host.das | 33 ++ tests/jit_tests/_jit_lib_shared_host.das | 35 ++ .../_jit_lib_two_module_sets_host.das | 32 ++ tests/jit_tests/jit_lib.das | 218 ++++++++++ tests/standalone-sweep/CMakeLists.txt | 96 +++++ tests/standalone-sweep/sweep_contexts.cmake | 49 +++ tests/standalone-sweep/sweep_driver.c | 30 ++ tests/standalone-sweep/sweep_jit.cmake | 112 +++++ utils/daslang/main.cpp | 24 ++ utils/internal/jit/main.das | 76 ++-- 59 files changed, 2305 insertions(+), 231 deletions(-) create mode 100644 examples/c_api_library/CMakeLists.txt create mode 100644 examples/c_api_library/greetings.das create mode 100644 examples/c_api_library/main.das create mode 100644 examples/c_api_library/shapes.das create mode 100644 examples/c_api_library/units.das create mode 100644 modules/dasLLVM/ARCHITECTURE_LIB.md create mode 100644 modules/dasLLVM/daslib/jit_standalone.das create mode 100644 tests-cpp/small/test_jit_lib_guard.cpp create mode 100644 tests/jit_tests/_jit_lib_align.das create mode 100644 tests/jit_tests/_jit_lib_align_host.das create mode 100644 tests/jit_tests/_jit_lib_bind.das create mode 100644 tests/jit_tests/_jit_lib_bind_host.das create mode 100644 tests/jit_tests/_jit_lib_export_all_host.das create mode 100644 tests/jit_tests/_jit_lib_guest.das create mode 100644 tests/jit_tests/_jit_lib_guest_host.das create mode 100644 tests/jit_tests/_jit_lib_probe.das create mode 100644 tests/jit_tests/_jit_lib_probe_host.das create mode 100644 tests/jit_tests/_jit_lib_shared_host.das create mode 100644 tests/jit_tests/_jit_lib_two_module_sets_host.das create mode 100644 tests/jit_tests/jit_lib.das create mode 100644 tests/standalone-sweep/CMakeLists.txt create mode 100644 tests/standalone-sweep/sweep_contexts.cmake create mode 100644 tests/standalone-sweep/sweep_driver.c create mode 100644 tests/standalone-sweep/sweep_jit.cmake diff --git a/.gitignore b/.gitignore index d2bca6bc67..0c6e6b85d8 100644 --- a/.gitignore +++ b/.gitignore @@ -50,6 +50,9 @@ _aot_generated/ _llvm_aot_generated/ # the -ctx driver writes each fixture's daslang bindings beside the host that requires them tests-cpp/big/standalone_ctx/_*_c.das +_standalone_ctx_generated/ +# the -lib suite generates each fixture's daslang bindings beside its host +tests/jit_tests/_*_c.das .vscode/ .cache/ @@ -201,3 +204,7 @@ modules/dasLLAMA/benchmarks/asr/_pybench_rows.txt # python bytecode, anywhere __pycache__/ site/files/profile_results_*.json + +# examples/c_api_library builds its three libraries here, and CMake generates their bindings +examples/c_api_library/_out/ +examples/c_api_library/*_c.das diff --git a/CLAUDE.md b/CLAUDE.md index e58d49517f..b5317876c0 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -98,7 +98,7 @@ Task-specific instructions are split into skill files under `skills/`. You MUST | `skills/internal/documentation_rst.md` | Editing RST in `doc/source/`, `//!` doc-comments in `daslib/*.das`, tutorial RST pages | | `skills/internal/tutorials.md` | Anything that looks like a tutorial - they live under `/tutorials//`, NEVER `modules//tutorial/` | | `skills/internal/tutorial_prose.md` | WRITING or revising general-reader doc/tutorial prose (`documentation_rst.md` is mechanics, this is the words) | -| `skills/cpp_integration.md` | Embedding daslang in C++; binding types/functions/enums; shipping without the compiler (`libDaScriptNano`, or a standalone context on the full runtime) | +| `skills/cpp_integration.md` | Embedding daslang in C++; binding types/functions/enums; shipping without the compiler (`libDaScriptNano`, or a standalone context on the full runtime); calling daslang from C (`daslang -lib`) | | `skills/internal/cpp_codebase_notes.md` | Working on daslang's own C++ - where inference/builtins/errors/parser live, AST function flags | | `skills/internal/clang_bind_build.md` | Enabling `dasClangBind` / bumping the libclang SDK / running any `bind_*.das` self-binder | | `skills/daslib_modules.md` | Working with `daslib/` modules or extending the stdlib | diff --git a/CMakeLists.txt b/CMakeLists.txt index 55e991d21e..0f245dcf9c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1645,6 +1645,9 @@ if (NOT ${DAS_TOOLS_DISABLED}) # their C++, which is why they live here and not beside the nano library. add_subdirectory(examples/standalone) + # `daslang -lib` libraries bound back into daslang through generated [extern] bindings. + add_subdirectory(examples/c_api_library) + endif() # This list should be significantly reduced, most of the files, except aot related should be private. diff --git a/daslib/ARCHITECTURE_CAPI.md b/daslib/ARCHITECTURE_CAPI.md index 74fcd14c6a..896c24862f 100644 --- a/daslib/ARCHITECTURE_CAPI.md +++ b/daslib/ARCHITECTURE_CAPI.md @@ -1,39 +1,52 @@ # daslib architecture notes - the generated C API Companion to `ARCHITECTURE.md` in this folder; section numbers are unique across the family. -The header this section describes is the one a standalone context ships. +The header this section describes is the one BOTH backends write: `aot_standalone` for a +standalone context, `inject_lib` for `daslang -lib`. ## 30. c_api_header -- **One module writes the generated header, and it owns the C declarations in it.** The C - declarations come first; the C++ half (`CppApi`) follows under `#ifdef __cplusplus`, and both +- **One module writes every generated header, and one module owns the C declarations in it.** The + C declarations come first; the C++ half (`CppApi`) follows under `#ifdef __cplusplus`, and both hosts include one file without seeing the other's half. `aot_standalone` emits the matching - `extern "C"` bodies off this describer, so the header and the source cannot disagree about what - crosses or what it is called. -- **C sits ON TOP of C++, not under it.** The C++ methods are DEFINED in the generated `.cpp` and - call the AOT functions directly; each `extern "C"` entry point is a wrapper over the method - beside it. So the C++ API is the wide one - every export, in native C++ types, spelled by - `aot_standalone` because it owns C++ types the way this module owns C ones - and C is the subset - C can express, not a ceiling over it. A method keeps the daslang name: `[export_c(name = ...)]` - renames the C symbol only. + `extern "C"` bodies and `inject_lib` the matching thunks, both off this describer, so JIT and + AOT cannot disagree about what crosses or what it is called. +- **C sits ON TOP of C++, not under it, and only in the AOT tier.** A standalone context's C++ + methods are DEFINED in the generated `.cpp` and call the AOT functions directly; each + `extern "C"` entry point is a wrapper over the method beside it. So the C++ API is the wide one - + every export, in native C++ types, spelled by `aot_standalone` because it owns C++ types the way + this module owns C ones - and C is the subset C can express, not a ceiling over it. A method + keeps the daslang name: `[export_c(name = ...)]` renames the C symbol only. `daslang -lib` cannot + follow, because a jitted library has no C++ source to put underneath: its C++ half stays inline + proxies over C, so the two tiers' C++ APIs differ ON PURPOSE, and only their C surface is twinned. - **The generated `.cpp` carries the header's text instead of including it.** The header is the HOST's file - it may move it or edit it - and the implementation must not be breakable that way. Types are declared once (`type_defs` never reaches the bodies), so inlining is a substitution for the include rather than a second copy. A host header that drifts from the library still fails loudly: a changed or removed signature is a link error, a changed layout trips the header's own size and offset asserts. -- **`Function.flags.exports` is the whole selection truth.** `[export_c]` (`ExportCAnnotation`, - `daslib/export_c.das` - a das `[function_macro]`, so the C surface is decided entirely in - daslang) sets it, and so does `[export]`, so a standalone context takes the bit however it was - set. Accepting the bit however it was set is what licenses skipping an unspellable signature with - a warning; a function that ASKED for C with `[export_c]` and cannot cross is an error. No second - list can drift from the bit. Whether a signature CAN cross is decided here instead, after infer, - because argument types do not exist when an annotation applies. -- **Refusal is per stage, not per module**: `collect_c_exports` returns its rejections and - logs its skips. An `[export_c]` that cannot cross comes back for the caller to report - - `macro_error` during compilation, the jit error log during codegen - so this module needs - no `ProgramPtr` and no reporting policy of its own. A merely-public function is skipped - with a warning naming it and the type. +- **`Function.flags.exports` is the whole selection truth.** `[export_c]` + (`ExportCAnnotation`, `daslib/export_c.das` - a das `[function_macro]`, so the C surface is + decided entirely in daslang) sets it, and + `policies.export_public_functions` sets it for every public entry-module function under + `-lib-export-all` (`MarkSymbolUse::exportPublicFunctions`, `src/ast/ast_export.cpp`). The + selection forms are therefore one bit read several ways: `-lib` alone accepts only what carries + the annotation, `--jit-lib-export-marked` and a standalone context accept the bit however it was + set (so `[export]` selects), and `-lib-export-all` is that same acceptance plus the marking + policy. Accepting the bit however set is what licenses skipping an unspellable signature with a + warning; one that ASKED for C with `[export_c]` and cannot cross is an error. Whether a signature + CAN cross is decided here, after infer, because argument types do not exist at annotation time. +- **`[export_c]` reaches a library source with no `require` because `daslib/export_c` is + `!inscope`.** That marker sets `visibleEverywhere`, which `Module::isVisibleDirectly` honors + ahead of the require map - the mechanism that makes `daslib/builtin.das` universal. Being visible + still needs the module LOADED, and `daslib/just_in_time.das` - injected whenever the JIT is on - + requires it, so `-lib`, `-jit` and `-exe` carry the annotation for free; a compile with no JIT + needs `require daslib/export_c`. It lives in its own module rather than in `c_api_header` because + that require costs `ast_boost` alone, not the header emitter's whole graph. +- **Refusal is per stage, not per module**: `collect_c_exports` returns its rejections and logs + its skips. An `[export_c]` that cannot cross comes back for the caller to report - `macro_error` + during compilation, the jit error log during codegen - so this module needs no `ProgramPtr` and + no reporting policy of its own. - **The scalar widths and the vector layouts are C++-side facts this emitter mirrors.** `bool` is one byte (the `static_assert(sizeof(bool)==1)` in `getTypeBaseSize`, `src/simulate/debug_info.cpp`), so das `bool` meets C as `bool`. `float3` is `{x, y, z}` at @@ -63,6 +76,17 @@ The header this section describes is the one a standalone context ships. all - while a representable target is defined, so a host can read through the pointer. A cycle terminates either way, and a pointer to something C cannot even name degrades to `void *` rather than refusing the function. -- **A `fixed_array` argument crosses (as `const T *`, which is what the das ABI already - passes) but a `fixed_array` result does not** - that would be a CMRES of an array, a - pointer nothing on the C side sizes. +- **The same describer writes the daslang twin of the header.** `build_das_bindings` + (`--jit-lib-bindings`) emits one `[extern(cdecl, late, ...)]` per entry point plus a das struct + per crossing structure, so a das host requires what the generator wrote instead of hand-writing + declarations or parsing the C header through clang. `late` is what lets the file compile before + the library exists, which is what a build system needs to generate it as an ordinary output. It + re-exports `dasbind` (`require dasbind public`), because the declarations land in that module and + a host requiring only the bindings would not see them. Types are spelled as das names them - + `float3`, `range`, `int` for an enumeration - so a host passes `safe_addr` of the real type and + writes no `reinterpret`; a type das cannot restate as the C ABI carries it degrades that one + entry point to a comment, not the file. Only `-lib` writes bindings: a standalone context emits + `extern "C"` bodies for the host to compile IN, with no export decoration, so a shared library + built from them exports nothing for dasbind to load. +- **A `fixed_array` argument crosses (as `const T *`, which the das ABI already passes) but a + `fixed_array` result does not** - that is a CMRES of an array, a pointer nothing in C sizes. diff --git a/doc/source/reference/embedding/advanced.rst b/doc/source/reference/embedding/advanced.rst index 3ff5ff9dd2..1da3adea18 100644 --- a/doc/source/reference/embedding/advanced.rst +++ b/doc/source/reference/embedding/advanced.rst @@ -268,7 +268,7 @@ Pipeline: #include "standalone_ctx_generated/script.das.h" - das::standalone::Standalone ctx; + das::ctx_standalone::Standalone ctx; ctx.test(); // direct call, no findFunction needed The same generated files also carry a C API: one ``extern "C"`` entry @@ -333,6 +333,13 @@ The generated header asserts the layout of every structure it declares (``_Static_assert`` in C11, ``static_assert`` in C++), so a host built for a different target fails to compile rather than misreading memory. +A daslang host needs no C: ``-- --jit-lib-bindings build/script_c.das`` +writes the daslang twin of the header beside it, one +``[extern(cdecl, late, ...)]`` per entry point plus a das struct per +structure that crosses, and the host ``require``\ s that file. The externs +bind ``late``, so the bindings compile before the library exists and a +build system can generate them as an ordinary output. + Several such libraries coexist in one process, and so does a library inside a host that registered the daslang modules itself: whoever gets there first registers the runtime, and the rest bind to it. Several instances of one diff --git a/doc/source/reference/embedding/c_api.rst b/doc/source/reference/embedding/c_api.rst index 12bfc739ea..68e81e1e78 100644 --- a/doc/source/reference/embedding/c_api.rst +++ b/doc/source/reference/embedding/c_api.rst @@ -37,6 +37,12 @@ functionality. The C API covers the most common embedding scenarios but does not expose every C++ feature (e.g. class adapters, custom annotations). +This API embeds the daslang *compiler*: the host loads sources, compiles +them, and calls what it finds by name. A host that only needs to call +one fixed script wants ``daslang -lib`` instead — it compiles that script +to a native library with its own generated C header, and the host links +no daslang API at all. See :ref:`embedding_advanced` (C libraries). + Linking ======= diff --git a/doc/source/reference/language/annotations.rst b/doc/source/reference/language/annotations.rst index 31111aa93f..b06c15f9a3 100644 --- a/doc/source/reference/language/annotations.rst +++ b/doc/source/reference/language/annotations.rst @@ -66,8 +66,9 @@ Lifecycle } ``[export_c]`` - An ``[export]`` that also names the function to C: a standalone context declares it in the C - half of the header it generates, so a C host calls it with no daslang API of its own. Needs + An ``[export]`` that ``daslang -lib`` additionally surfaces as a C function in the header it + generates. ``-lib``, ``-jit`` and ``-exe`` carry the annotation themselves, so a library source + needs no ``require``; a plain interpreter or AOT compile of the same file needs ``require daslib/export_c``. The function keeps working in every tier. Its signature has to be one C can spell: scalars, ``string``, raw pointers, enumerations, plain structures, the ``float2``..``uint4`` and ``range`` families, and ``fixed_array`` arguments; ``array``, diff --git a/doc/source/stdlib/handmade/structure_annotation-rtti-CodeOfPolicies.rst b/doc/source/stdlib/handmade/structure_annotation-rtti-CodeOfPolicies.rst index 084f1782e1..3a5b89501b 100644 --- a/doc/source/stdlib/handmade/structure_annotation-rtti-CodeOfPolicies.rst +++ b/doc/source/stdlib/handmade/structure_annotation-rtti-CodeOfPolicies.rst @@ -14,6 +14,7 @@ Whether we are in lint-check mode (standalone linters set this so modules can ad Skip Program::lint() entirely (as if every module set ``options lint = false``). Skip the Module::Initialize() assert in compileDaScript (for environments initialized later, e.g. dynamic-module discovery). Export all functions and global variables. +Treat every public, non-generic function of the entry module as [export] (daslang -lib -lib-export-all). If not set, we recompile main module each time. Keep context alive after main function. Whether to use very safe context (delete of data is delayed, to avoid table[foo]=table[bar] lifetime bugs). diff --git a/examples/c_api_library/CMakeLists.txt b/examples/c_api_library/CMakeLists.txt new file mode 100644 index 0000000000..44b130ceed --- /dev/null +++ b/examples/c_api_library/CMakeLists.txt @@ -0,0 +1,57 @@ +########################################################### +# Three daslang libraries with a C ABI, bound back into daslang. +# +# Each .das becomes a shared library through `daslang -lib`, and the same run writes the +# daslang `[extern]` bindings for it - the das twin of the generated C header. main.das +# requires those, so the example hand-writes no declaration and parses no C header. +# +# The bindings sit beside their source because a das `require` resolves relative to the +# requiring file; they are generated, and .gitignore covers them. +########################################################### + +if(DAS_LLVM_DISABLED) + return() +endif() + +# The libraries live beside the source, the one place main.das also rebuilds them into, and +# the -output path stays relative to the tree root so the bindings carry a relative library +# path rather than this machine's absolute one. +set(C_API_LIB_OUT "examples/c_api_library/_out") + +function(das_c_api_library stem select_flag jit_flag) + set(_bind "${CMAKE_CURRENT_SOURCE_DIR}/${stem}_c.das") + # one -lib run writes both, and main.das rebuilds neither: declare the library too, or a + # deleted artifact leaves the example bound to nothing + set(_lib "${CMAKE_CURRENT_SOURCE_DIR}/_out/${stem}${CMAKE_SHARED_LIBRARY_SUFFIX}") + add_custom_command( + OUTPUT "${_bind}" "${_lib}" + COMMAND ${CMAKE_COMMAND} -E make_directory "${CMAKE_CURRENT_SOURCE_DIR}/_out" + COMMAND $ -lib "${CMAKE_CURRENT_SOURCE_DIR}/${stem}.das" + -output "${C_API_LIB_OUT}/${stem}" ${select_flag} + -- ${jit_flag} --jit-lib-bindings "${_bind}" + DEPENDS daslang + "${CMAKE_CURRENT_SOURCE_DIR}/${stem}.das" + "${PROJECT_SOURCE_DIR}/daslib/c_api_header.das" + "${PROJECT_SOURCE_DIR}/modules/dasLLVM/daslib/jit_standalone.das" + "${PROJECT_SOURCE_DIR}/modules/dasLLVM/daslib/llvm_jit_run.das" + WORKING_DIRECTORY "${PROJECT_SOURCE_DIR}" + COMMENT "daslang -lib bindings: ${stem}" + VERBATIM + ) + set(C_API_LIB_BINDINGS ${C_API_LIB_BINDINGS} "${_bind}" PARENT_SCOPE) +endfunction() + +das_c_api_library(shapes "" "") +das_c_api_library(units -lib-export-all "") +das_c_api_library(greetings "" --jit-lib-export-marked) + +# ALL, not just a test-small dependency: CI runs `ctest -L small` without building that target +add_custom_target(c_api_library_bindings ALL DEPENDS ${C_API_LIB_BINDINGS}) +set_target_properties(c_api_library_bindings PROPERTIES FOLDER "examples") + +# The libraries and their bindings both come from the commands above; main.das builds nothing, +# so the example is only about binding three libraries at once and calling them. +add_test(NAME c_api_library COMMAND $ examples/c_api_library/main.das + WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}) +set_tests_properties(c_api_library PROPERTIES LABELS "small") +add_dependencies(test-small c_api_library_bindings) diff --git a/examples/c_api_library/greetings.das b/examples/c_api_library/greetings.das new file mode 100644 index 0000000000..71243a32c6 --- /dev/null +++ b/examples/c_api_library/greetings.das @@ -0,0 +1,17 @@ +options gen2 +options indenting = 4 + + +[export] +def greet(who : string) : string { + return "hola {who}" +} + +[export] +def bump(n : int) : int { + return n + 1 +} + +def internal_seed() : int { + return 41 +} diff --git a/examples/c_api_library/main.das b/examples/c_api_library/main.das new file mode 100644 index 0000000000..3a75702611 --- /dev/null +++ b/examples/c_api_library/main.das @@ -0,0 +1,43 @@ +options gen2 +options indenting = 4 + + +require daslib/safe_addr + +require shapes_c +require units_c +require greetings_c + +[export] +def main() : int { + var sh = shapes_create() + var un = units_create() + var gr = greetings_create() + if (sh == null || un == null || gr == null) { + print("a library refused to create an instance: {shapes_last_error(null)}\n") + return 1 + } + var a = shapes_Vec2(x = 3.0, y = 4.0) + var b = shapes_Vec2(x = 1.0, y = 2.0) + let dot = shapes_dot(sh, safe_addr(a), safe_addr(b)) + let fahrenheit = units_celsius_to_fahrenheit(un, 100.0) + let greeting = greetings_greet(gr, "mundo") + print("shapes.dot((3,4),(1,2)) = {dot}\n") + print("units.celsius_to_fahrenheit(100) = {fahrenheit}\n") + print("greetings.greet(\"mundo\") = {greeting}\n") + greetings_destroy(gr) + units_destroy(un) + shapes_destroy(sh) + var bad = 0 + if (dot != 11.0) { + bad++ + } + if (fahrenheit != 212.0) { + bad++ + } + if (greeting != "hola mundo") { + bad++ + } + print(bad == 0 ? "three daslang libraries, one process - ok\n" : "{bad} of three answered wrong\n") + return bad +} diff --git a/examples/c_api_library/shapes.das b/examples/c_api_library/shapes.das new file mode 100644 index 0000000000..1f2ac59a14 --- /dev/null +++ b/examples/c_api_library/shapes.das @@ -0,0 +1,24 @@ +options gen2 +options indenting = 4 + + +require daslib/export_c + +struct Vec2 { + x : float + y : float +} + +[export_c] +def dot(a, b : Vec2) : float { + return a.x * b.x + a.y * b.y +} + +[export_c(name = "scaled")] +def scale(v : Vec2; k : float) : Vec2 { + return Vec2(x = v.x * k, y = v.y * k) +} + +def length_squared(v : Vec2) : float { + return dot(v, v) +} diff --git a/examples/c_api_library/units.das b/examples/c_api_library/units.das new file mode 100644 index 0000000000..8acede2724 --- /dev/null +++ b/examples/c_api_library/units.das @@ -0,0 +1,17 @@ +options gen2 +options indenting = 4 + +require math + + +def celsius_to_fahrenheit(c : float) : float { + return c * 1.8 + 32.0 +} + +def clamp_int(v, lo, hi : int) : int { + return clamp(v, lo, hi) +} + +def private rounding_bias() : float { + return 0.5 +} diff --git a/include/daScript/ast/dyn_modules.h b/include/daScript/ast/dyn_modules.h index cef5660316..984da1aa51 100644 --- a/include/daScript/ast/dyn_modules.h +++ b/include/daScript/ast/dyn_modules.h @@ -45,6 +45,7 @@ DAS_API void replay_dynamic_module(const char * path, const char * cpp_class, in DAS_API void defer_dynamic_module(const char * path, const char * cpp_class, int on_error, const char * das_name); DAS_API bool load_deferred_dynamic_module(const char * das_name); DAS_API size_t load_all_deferred_dynamic_modules(); // the count it attempted +DAS_API string describe_pending_dynamic_modules(); // "" when nothing is pending DAS_API bool is_dynamic_module_deferred(const char * das_name); DAS_API bool pending_dynamic_module_artifact_present(); // a dlopen-failed row whose file exists: an import the deferred set holds DAS_API string registered_dynamic_module_name(const char * path, const char * cpp_class); // empty while that row's module is not registered diff --git a/include/daScript/simulate/code_of_policies.h b/include/daScript/simulate/code_of_policies.h index 78bb940b67..2d27daf29f 100644 --- a/include/daScript/simulate/code_of_policies.h +++ b/include/daScript/simulate/code_of_policies.h @@ -41,6 +41,7 @@ namespace das { bool no_lint = false; // skip Program::lint() entirely bool no_init_check = false; // skip the Module::Initialize() assert, most of the time should be false (except maybe dynamic-module discovery) bool export_all = false; // when user compiles, export all (public?) functions + bool export_public_functions = false; bool serialize_main_module = true; // if false, then we recompile main module each time bool keep_alive = false; // produce keep-alive noodes /*option*/ bool very_safe_context = false; // context is very safe (does not release old memory from array or table grow, leaves it to GC) diff --git a/install/CLAUDE.md b/install/CLAUDE.md index 9d83ff71a1..16a77746cc 100644 --- a/install/CLAUDE.md +++ b/install/CLAUDE.md @@ -53,7 +53,7 @@ Task-specific instructions are in skill files under `skills/`. Read the relevant | `skills/mcp_tools.md` | Full MCP tool table + live-API reference | | `skills/das_formatting.md` | Creating or modifying any `.das` file | | `skills/comment_style_hygiene.md` | Writing or reviewing comments, names, or local code shape in ANY language | -| `skills/cpp_integration.md` | Embedding daslang in C++; binding types/functions/enums; shipping without the compiler (`libDaScriptNano`, or a standalone context on the full runtime) | +| `skills/cpp_integration.md` | Embedding daslang in C++; binding types/functions/enums; shipping without the compiler (`libDaScriptNano`, or a standalone context on the full runtime); calling daslang from C (`daslang -lib`) | | `skills/daslib_modules.md` | Using `daslib/` modules (linq, json, regex, etc.) | | `skills/das_macros.md` | Compile-time macros, AST manipulation, qmacro/quote, gc_node patterns | | `skills/daspkg.md` | Creating `.das_package` manifests, daspkg commands | diff --git a/modules/dasLLVM/.das_module b/modules/dasLLVM/.das_module index 413bcb6b09..b80c5f3a96 100644 --- a/modules/dasLLVM/.das_module +++ b/modules/dasLLVM/.das_module @@ -9,7 +9,7 @@ def initialize(project_path : string) { "llvm_dsl", "llvm_jit_intrin", "llvm_jit_common", "llvm_jit_lower", "llvm_dll_utils", "llvm_jit_di", - "llvm_exe", "llvm_macro", "llvm_jit_cli", "llvm_jit_run", "llvm_aot", + "llvm_exe", "jit_standalone", "llvm_macro", "llvm_jit_cli", "llvm_jit_run", "llvm_aot", "llvm_jit_plan", // the emitter-free half of the driver: plan, DLL key, probe, install "llvm_jit_link", // the JIT entry: a DLL cache hit binds here, a miss brings llvm_macro in "llvm_env", // [EnvConfig] environment-knob registry (ENVIRONMENT.md generates from it) diff --git a/modules/dasLLVM/ARCHITECTURE.md b/modules/dasLLVM/ARCHITECTURE.md index 3ab092a30b..e0fd51e3e1 100644 --- a/modules/dasLLVM/ARCHITECTURE.md +++ b/modules/dasLLVM/ARCHITECTURE.md @@ -16,10 +16,11 @@ program: **hash** (the DLL key over the candidate set: per-function AOT hashes p folds), **init** (engine + target machine), **declare** (LLVM function declarations for the jit set), **probe** (open the cached DLL, compare per-function hashes), **irgen** (the das IR emitter over every function), **optimize** (the LLVM pass pipeline at the requested level, plus the -opt-in IR dump and the post-opt verify), **emit+link** (artifact production - for the DLL path -`write_dll` in `llvm_jit_common.das`, itself **emit-obj**, machine-code emission, then **link**, -the lld-link spawn), **install** (resolve externs, instrument sim nodes), and **finalize** -(engine teardown / state install). On a hit irgen, optimize and emit+link read as zero. +opt-in IR dump and the post-opt verify), **emit+link** (artifact production - `write_artifact` in +`llvm_jit_common.das`, itself **emit-obj**, machine-code emission, then **link**; which artifact +is the caller's choice, a JIT DLL or an exe or, for `-lib`, a shared library or static archive +with the C header written beside it), **install** (resolve externs, instrument sim nodes), and +**finalize** (engine teardown / state install). On a hit irgen, optimize and emit+link read as zero. ### 1.1 The timing contract @@ -57,13 +58,17 @@ one of those files visible (re-pin `LLVM_JIT_EMITTER_HASH`), and the bump is owe emitted code for identical inputs can differ - a comment, a nolint, or a same-value rewrite inside an emitter file re-pins without a bump. -`--jit-opt-level` (CLI, over `policies.jit_opt_level`, default 3) drives both the optimize -pipeline and the DLL path's codegen-side target machine. `write_exe` and AOT-object emission -(`emit_object_only`) deliberately stay at codegen level 3: shipped artifacts are not -content-addressed, so a tier change there has no cache guard to catch it. At level 0 the -injected tune-policy default becomes `fallback` (`jit_cli_opt_level()` in `llvm_tune.das`): -tune winners are raced under O3 codegen, so an O0 run cannot represent them and must not block -on the tuner to mint them. +The per-artifact entry emitters are OUTSIDE that surface: `llvm_exe.das` (a standalone exe's +`main`) and its `-lib` half (the C entry points and thunks) emit startup glue for artifacts +nothing content-addresses, so neither is in `EMITTER_FILES` and neither owes a version bump. Both +artifacts run in `LlvmJitMode.EXE`, which is what makes the exe startup shareable: the four +`emit_standalone_*` helpers in `llvm_exe.das` are the shared halves, split so a library can put the +process-global half behind a once guard and the per-context half behind its own catch boundary. + +### 1.3 A library's runtime + +`daslang -lib` emits an artifact that loads into a process it does not own; its runtime, +environment and shutdown rules are `ARCHITECTURE_LIB.md` sec. 1.3. ## 2. Codegen identity - the DLL cache diff --git a/modules/dasLLVM/ARCHITECTURE_LIB.md b/modules/dasLLVM/ARCHITECTURE_LIB.md new file mode 100644 index 0000000000..4f7f801e50 --- /dev/null +++ b/modules/dasLLVM/ARCHITECTURE_LIB.md @@ -0,0 +1,67 @@ +# dasLLVM Architecture - the emitted library + +Companion to `ARCHITECTURE.md` in this folder; section numbers are unique across both files. + +### 1.3 A library's runtime is process-global, its environment is per-thread {#lib-runtime-scope} + +An exe owns its process: one thread runs `main`, registers the modules, and drains them on the way +out. A library owns none of that, and three consequences shape its entry points. + +**A library's process state lives in the library, not in the runtime.** `inject_lib` emits three +private globals - the init guard, the environment it bound, and the text of a create that returned +no instance - and passes them to the runtime shims by pointer. So the shim layer holds no state of +its own, two libraries in one process share none, and nothing needs a per-library table. + +**The environment is thread-local** (`daScriptEnvironment::bound` / `owned`, +`include/daScript/ast/ast.h`), but the module registration behind it happens once. So +`jit_lib_run_once(guard, env, init)` does both jobs: the first caller registers and records its +environment, and every later caller - on any thread - is BOUND to that recording. Without the +binding a second thread's `

_create` dereferences a null `getBound()->modules` inside +`jit_init_extern_function`. The create-failure text sits in the library's slot for the same reason +read from the other side: a C host reads it through `

_last_error(nullptr)`, which has no +instance to ask, and a thread-local copy would answer null on every thread but the failing one. + +**One daslang runtime fits in a process, and every artifact in it shares that one.** +`jit_register_Module_*` (`REGISTER_MODULE_IN_NAMESPACE`) carries no already-created guard and +aborts on a second call, so every emitted registration goes through +`jit_register_module_once(dasName, reg)`, which registers only when `Module::require(dasName)` +finds nothing and otherwise hands back the module already there. EVERY site means both of them - +the require walk and the extern collector's `ensure_module`, which reaches a module no `require` +names (an `ast_core` extern called from a library aborted the host until it did). A library that finds a populated +environment is a GUEST, and the guard records which it is - 1 for the library that registered the +runtime, 2 for one that joined a populated one, decided in `jit_lib_run_once` before the init call. +A guest skips `Module::Initialize` so `g_envTotal` stays balanced, and it never drains what it did +not create. Both facts are read back off the guard: `jit_initialize_modules_done` and +`jit_lib_shutdown` take its value, so ownership needs no second flag and a library arriving later +cannot erase an earlier one's. + +**A guest reports a failed dynamic module; it does not abort.** `jit_finalize_dynamic_modules` +fatals on anything still unloadable, which is right for an exe that owns its process and wrong for +a library: the pending list it inherits is the HOST's, and a module the host could not load is not +this library's to kill the process over. A guest calls `jit_lib_finalize_dynamic_modules` instead - +retry, report, carry on - and a module this library actually needs still surfaces, as a null +`

_create` carrying the reason. + +**Nothing calls the shutdown functions for a library.** A jitted `SimFunction` gets a zeroed +`FuncInfo` (`jit_lib`'s `registerJitFunction`, `src/builtin/module_jit.cpp`), so +`Context::runShutdownScript` - which selects on `FuncInfo::flag_shutdown`, set only by the +interpreter's debug-info builder - finds none. `

_destroy` therefore emits the program's +`[finalize]` / `[shutdown]` calls itself, the same way `

_create` emits its `[init]` calls. The +process-level drain is the host's explicit, final `

_shutdown_runtime()` - never automatic, since +a library cannot know whether the process is done with daslang, and an `atexit` hook would need +process-global state to decide. It is a no-op unless this library's guard says it owns the runtime. + +Every linked artifact leaves through one emitter, `write_artifact` (`llvm_jit_common.das`): the +`JitArtifact` kind picks the file name, the linker flavor (an archiver for a static library, no +`-shared` for an exe) and the target CPU. A `jit_dll` always targets this host - it only ever runs +on the box that emitted it - while a shipped artifact takes the caller's `use_host_cpu`: generic +and redistributable by default, host-specific only when the build carries `[llvm_code]` kernels, +whose tuner-generated IR a generic target refuses to legalize. + +`--jit-opt-level` (CLI, over `policies.jit_opt_level`, default 3) drives both the optimize +pipeline and the DLL path's codegen-side target machine. A shipped artifact and AOT-object +emission (`emit_object_only`) deliberately stay at codegen level 3: shipped artifacts are not +content-addressed, so a tier change there has no cache guard to catch it. At level 0 the +injected tune-policy default becomes `fallback` (`jit_cli_opt_level()` in `llvm_tune.das`): +tune winners are raced under O3 codegen, so an O0 run cannot represent them and must not block +on the tuner to mint them. diff --git a/modules/dasLLVM/README.md b/modules/dasLLVM/README.md index cd29e509c4..3c2bbec2d6 100644 --- a/modules/dasLLVM/README.md +++ b/modules/dasLLVM/README.md @@ -81,6 +81,72 @@ cached DLL for instant execution. - By default, the `dll` is stored in `.jitted_scripts/`. - This can be changed using `jit_output_path`. +## Native library with a C API (`-lib`) +`-lib` emits a native library plus the C header a host calls it through, so a +program that only needs to *call* one script links no daslang API: + +```sh +./bin/daslang -lib script.das -output build/script +``` + +writes `build/script.so` (`.dylib` / `.dll`) and `build/script.h`. Add +`-- --jit-lib-static` for a `.a` / `.lib` archive instead; the generated header +records what a host then links. + +Which functions cross the boundary, in three forms: `[export_c]` marks them one +at a time; `-- --jit-lib-export-marked` takes whatever the program already marks +`[export]`, so a script with a host API needs no new annotation; and +`-lib-export-all` offers every public function of the entry module whose +signature C can spell (naming the ones it skips). The first two are a selection +only - a signature C cannot spell stays a hard error - while export-all skips it +with a warning. `examples/c_api_library/` builds one library each way and loads +all three at once through dasbind. + +`-- --jit-lib-bindings out/script_c.das` writes the daslang twin of the header +alongside it: one `[extern(cdecl, late, ...)]` per entry point, plus a das struct +per structure that crosses. A daslang host then `require`s that file instead of +hand-writing the declarations or binding the C header through clang, and because +the externs are `late` the bindings compile before the library exists - which is +what lets a build system generate them as an ordinary output. The example's +CMakeLists does exactly that for its three libraries. Every library gets the same four entry points, +prefixed with the output name, so they always match the header they are declared in: + +```c +script_ctx * script_create(void); +void script_destroy(script_ctx * ctx); +const char * script_last_error(script_ctx * ctx); +void script_shutdown_runtime(void); +``` + +One instance owns one daslang context - its own globals, heap and string heap - and `_create` +works on any thread. Several such libraries coexist in one process, as does one inside a host that +registered the daslang modules itself: the first there registers the runtime, the rest bind to it. +Scalars, `string`, pointers and enums cross by value; structures and the +`float2`..`uint4` / `range` families cross as `const T *` and return through a +trailing `T * out`. A daslang panic returns zero, leaves `out` untouched, and +shows up in `script_last_error(ctx)` until the next call clears it. + +Codegen is the same as `-exe`: the same private wrappers, the same +load-resolved globals, no DLL cache. Cross-compilation is not supported here - +build the library on its target host. + +## Standalone contexts, and which tier writes one + +A standalone context is a script that carries its own `Context` and needs no +compiler at run time. Two tiers write one, from the same script: + +- **AOT** - `daslang utils/aot/main.das -- -ctx script.das out/` emits C++ the + host compiles itself: `class Standalone : public Context` plus one method per + `[export]` function, next to the same C entry points `-lib` emits - one + describer writes both. The host owns the build, so it cross-compiles + anywhere its own toolchain reaches. +- **LLVM** - `-lib` and `-exe` above emit the binary directly, no C++ compiler + in the loop and no daslang sources shipped. `-exe` can cross-compile, since + codegen picks the target triple; `-lib` is host-only. + +Both answer the same C API, so a host can move between them without editing a +call site. `skills/cpp_integration.md` covers choosing one. + ## Cross-compilation (WebAssembly) The JIT pipeline can emit a non-host target instead of running on the host. The supported cross-target is `wasm32-unknown-emscripten` (the default when no diff --git a/modules/dasLLVM/REVIEW.md b/modules/dasLLVM/REVIEW.md index 7b9c8659e5..995d4c6e96 100644 --- a/modules/dasLLVM/REVIEW.md +++ b/modules/dasLLVM/REVIEW.md @@ -76,6 +76,14 @@ there. (`ARCHITECTURE.md` sec.5). An unpinned compile-time file read serves stale macro output from the module cache until an unrelated source file changes - silently. +- **A diff that adds a `-lib` entry point, or work to one, keeps the C boundary's three promises: + a raise reaches the caller as a return value and never an unwind, `

_create` answers null + rather than aborting, and `

_destroy` runs what the runtime's own shutdown cannot find** + (`ARCHITECTURE_LIB.md#lib-runtime-scope`). A library is called by code that cannot catch anything. + +- **A `-lib` build that writes no artifact exits non-zero.** A build rule reads the exit code, and + a silent success lets it link the previous run's library against this run's header. + - **A change to a `[tune]`-family annotation is reviewed with `skills/tune.md`.** - **A change to the tune framework - `daslib/llvm_tune.das`, its tests, or the descriptor and diff --git a/modules/dasLLVM/daslib/jit_standalone.das b/modules/dasLLVM/daslib/jit_standalone.das new file mode 100644 index 0000000000..624aef656a --- /dev/null +++ b/modules/dasLLVM/daslib/jit_standalone.das @@ -0,0 +1,327 @@ +options gen2 +options indenting = 4 +options no_unused_block_arguments = false +options no_unused_function_arguments = false +options strict_smart_pointers = false +options relaxed_pointer_const +options unsafe_table_lookup = false +options no_global_variables = false + +module jit_standalone shared private + +require llvm/daslib/llvm_boost +require llvm/daslib/llvm_jit +require llvm/daslib/llvm_jit_common +require llvm/daslib/llvm_dll_utils +require llvm/daslib/llvm_dsl +require daslib/ast_boost +require daslib/option +require daslib/c_api_header +require daslib/safe_addr +require daslib/defer + + +def public lib_link_note(static_lib : bool; link_whole_lib : bool) : string { + let windows = get_platform_name() == "windows" + let rt_static = windows ? "libDaScript_runtime.lib" : "liblibDaScript_runtime.a" + let cc_static = windows ? "libDaScript.lib" : "liblibDaScript.a" + let rt_shared = windows ? "libDaScriptDyn_runtime.dll" : (get_platform_name() == "darwin" ? "liblibDaScriptDyn_runtime.dylib" : "liblibDaScriptDyn_runtime.so") + if (static_lib) { + let whole = link_whole_lib ? " Add /lib/{cc_static} too - this library\nregisters a module that lives in the compiler library." : "" + let sys = windows ? "" : " Also link the platform libraries the runtime needs:\n-lpthread -ldl -lm -lstdc++." + return "Link this archive plus /lib/{rt_static}.{whole}{sys}" + } + let whole = link_whole_lib ? " It also needs the compiler library beside it - this library\nregisters a module that lives there." : "" + return "Link this shared library. At load it needs {rt_shared}, which it looks for beside itself,\nin ../lib, and in /lib; otherwise put it on the loader path.{whole}" +} + + +def public write_das_bindings(var exports : array; names : CNames; + input_path, output_path, requested : string; static_lib : bool) : bool { + let bindings_path = requested == "auto" ? "{output_path}_c.das" : requested + if (static_lib) { + to_log(LOG_WARNING, "LLVM LIB: --jit-lib-bindings names a shared library - a static archive has nothing for dasbind to load\n") + } + let b = DasBindings(linux_path = "{output_path}.so", macos_path = "{output_path}.dylib", + windows_path = "{output_path}.dll") + let cmd = "daslang -lib {input_path} -output {output_path}" + if (!emit_das_bindings(exports, names, b, cmd, bindings_path)) { + to_log(LOG_ERROR, "LLVM LIB: the library is written but its daslang bindings are not - {bindings_path}\n") + return false + } + return true +} + + +def public new_function(ctx : LLVMContextRef; name : string; typ : LLVMTypeRef; exported : bool) : tuple { + let fn = LLVMAddFunctionWithType(g_mod, name, typ) + if (exported) { + set_public_linkage(fn) + } else { + set_private_linkage(fn) + } + let b = LLVMCreateBuilderInContext(ctx) + LLVMPositionBuilderAtEnd(b, LLVMAppendBasicBlockInContext(ctx, fn, "entry")) + return (fn = fn, builder = b) +} + + +struct private ThunkAbi { + arg_types : array + arg_loads : array + c_types : array + slot_type : LLVMOpaqueType? + has_slot : bool +} + + +//! The three type lists one thunk needs: what the impl takes (das ABI), what C hands over, and +//! the result slot. `arg_loads[i]` marks a C pointer the impl wants BY VALUE - a vector or a +//! range - so that argument arrives as `*(T *)incoming`, not as the pointer itself. +def private thunk_abi(var e : CExport; var types : PrimitiveTypes?) : ThunkAbi { + var abi : ThunkAbi + abi.c_types |> push(types.LLVMVoidPtrType()) + for (arg, p in e.fn.arguments, e.params) { + abi.arg_types |> push(type_to_llvm_abi_type(arg._type)) + abi.arg_loads |> push(p.by_pointer && !p.pointer_in_impl) + abi.c_types |> push(p.by_pointer ? types.LLVMVoidPtrType() + : (arg._type.isBool ? types.t_int8 : type_to_llvm_abi_type(arg._type))) + } + if (e.result.via_out) { + abi.c_types |> push(types.LLVMVoidPtrType()) + } + if (!e.fn.result.isVoid) { + abi.has_slot = true + abi.slot_type = (e.result.cmres ? types.LLVMVoidPtrType() : type_to_llvm_abi_type(e.fn.result)) + } + return <- abi +} + + +def private frame_type(var abi : ThunkAbi; ctx : LLVMContextRef) : LLVMOpaqueType? { + var fields <- [for (t in abi.arg_types); t] + if (abi.has_slot) { + fields |> push(abi.slot_type) + } + if (fields |> empty()) { + fields |> push(g_prim_t.t_int32) + } + return LLVMStructTypeInContext(ctx, array_data_ptr(fields), uint(length(fields)), 0) +} + + +def private emit_trampoline(ctx : LLVMContextRef; var e : CExport; var uids : UidNodes?; + var types : PrimitiveTypes?; var abi : ThunkAbi; ft : LLVMOpaqueType?) : LLVMOpaqueValue? { + let impl_name = uids.get_dll_fn_name_ptr(e.fn).impl() + var impl = LLVMGetNamedFunction(g_mod, impl_name) + if (impl == null) { + return null + } + let tramp_type = jit_fn_type($(ctx, frame : void?) : void {}) + let made = new_function(ctx, "__das_lib_tramp_{e.c_name}", tramp_type, false) + let b = made.builder + defer() { + LLVMDisposeBuilder(b) + } + var ctx_arg = LLVMGetParam(made.fn, 0u) + var frame = LLVMBuildPointerCast(b, LLVMGetParam(made.fn, 1u), LLVMPointerType(ft, 0u), "frame") + var args : array + args |> reserve(length(abi.arg_types) + 2) + for (at, i in abi.arg_types, count()) { + let slot = LLVMBuildStructGEP2(b, ft, frame, uint(i), "arg_{i}") + args |> push(LLVMBuildLoad2(b, at, slot, "argv_{i}")) + } + args |> push(ctx_arg) + let slot_index = uint(length(abi.arg_types)) + if (e.result.cmres) { + var slot_ptr = LLVMBuildStructGEP2(b, ft, frame, slot_index, "res_ptr") + args |> push(LLVMBuildLoad2(b, types.LLVMVoidPtrType(), slot_ptr, "res")) + } + var ret = LLVMBuildCall2(b, g_fn_types[impl_name], impl, args, "") + if (abi.has_slot && !e.result.cmres) { + LLVMBuildStore(b, ret, LLVMBuildStructGEP2(b, ft, frame, slot_index, "res")) + } + LLVMBuildRetVoid(b) + return made.fn +} + + +def private c_return_type(var e : CExport; var types : PrimitiveTypes?) : LLVMTypeRef { + if (e.result.via_out || e.fn.result.isVoid) { + return types.t_void + } + return e.fn.result.isBool ? types.t_int8 : type_to_llvm_abi_type(e.fn.result) +} + + +def private store_thunk_args(b : LLVMBuilderRef; var e : CExport; var abi : ThunkAbi; + fn : LLVMOpaqueValue?; ft : LLVMOpaqueType?; var types : PrimitiveTypes?; frame : LLVMOpaqueValue?) { + for (arg, p, i in e.fn.arguments, e.params, count()) { + var incoming = LLVMGetParam(fn, uint(i + 1)) + var slot = LLVMBuildStructGEP2(b, ft, frame, uint(i), "in_{i}") + if (abi.arg_loads[i]) { + var typed = LLVMBuildPointerCast(b, incoming, LLVMPointerType(abi.arg_types[i], 0u), "") + incoming = LLVMBuildLoadData2Aligned(b, abi.arg_types[i], typed, arg._type.alignOf, "val_{i}") + } elif (p.by_pointer) { + incoming = LLVMBuildPointerCast(b, incoming, abi.arg_types[i], "") + } elif (arg._type.isBool) { + incoming = LLVMBuildICmp(b, LLVMIntPredicate.LLVMIntNE, incoming, types.ConstI8(int8(0)), "b_{i}") + } + LLVMBuildStore(b, incoming, slot) + } +} + + +def public emit_thunk(ctx : LLVMContextRef; var e : CExport; var uids : UidNodes?; var types : PrimitiveTypes?; + export_all : bool) : Option { + var abi <- thunk_abi(e, types) + let ft = frame_type(abi, ctx) + let tramp = emit_trampoline(ctx, e, uids, types, abi, ft) + if (tramp == null) { + if (export_all) { + to_log(LOG_WARNING, "LLVM LIB: skipping `{e.fn.name}`: no jitted body to export to C\n") + return some(true) + } + to_log(LOG_ERROR, "LLVM LIB: no jitted body for `{e.fn.name}` - it cannot be exported to C\n") + return none(type) + } + if (LLVMGetNamedFunction(g_mod, e.c_name) != null) { + to_log(LOG_ERROR, "LLVM LIB: C symbol `{e.c_name}` is already emitted; rename one with [export_c(name = \"...\")]\n") + return none(type) + } + let made = new_function(ctx, e.c_name, LLVMFunctionType(c_return_type(e, types), abi.c_types), true) + let b = made.builder + defer() { + LLVMDisposeBuilder(b) + } + var frame = LLVMBuildAlloca(b, ft, "frame") + var cmres_slot : LLVMOpaqueValue? + if (e.result.cmres) { + cmres_slot = LLVMBuildAlloca(b, LLVMArrayType(types.t_int8, uint(e.fn.result.sizeOf)), "result") + LLVMSetAlignment(cmres_slot, uint(e.fn.result.alignOf)) + LLVMBuildStore(b, LLVMBuildPointerCast(b, cmres_slot, types.LLVMVoidPtrType(), ""), + LLVMBuildStructGEP2(b, ft, frame, uint(length(abi.arg_types)), "res_ptr")) + } + store_thunk_args(b, e, abi, made.fn, ft, types, frame) + let guard_type = jit_fn_type($(ctx, tramp, frame : void?) : int {}) + var guard = declare_extern_fn("jit_lib_invoke_guarded", guard_type) + var guard_args = array(LLVMGetParam(made.fn, 0u), + LLVMBuildPointerCast(b, tramp, types.LLVMVoidPtrType(), ""), + LLVMBuildPointerCast(b, frame, types.LLVMVoidPtrType(), "")) + var ok = LLVMBuildCall2(b, guard_type, guard, guard_args, "ok") + if (!abi.has_slot) { + LLVMBuildRetVoid(b) + return some(false) + } + let ok_bb = LLVMAppendBasicBlockInContext(ctx, made.fn, "ok") + let fail_bb = LLVMAppendBasicBlockInContext(ctx, made.fn, "raised") + var cond = LLVMBuildICmp(b, LLVMIntPredicate.LLVMIntNE, ok, types.ConstI32(0ul), "ran") + LLVMBuildCondBr(b, cond, ok_bb, fail_bb) + let slot_index = uint(length(abi.arg_types)) + LLVMPositionBuilderAtEnd(b, ok_bb) + if (e.result.via_out) { + var out_ptr = LLVMGetParam(made.fn, uint(length(e.params) + 1)) + var source = e.result.cmres ? cmres_slot : LLVMBuildStructGEP2(b, ft, frame, slot_index, "res") + LLVMBuildMemCpy(b, out_ptr, 1u, source, uint(e.fn.result.alignOf), types.ConstI64(uint64(e.fn.result.sizeOf))) + LLVMBuildRetVoid(b) + } else { + var value = LLVMBuildLoad2(b, abi.slot_type, LLVMBuildStructGEP2(b, ft, frame, slot_index, "res"), "value") + LLVMBuildRet(b, e.fn.result.isBool ? LLVMBuildZExt(b, value, types.t_int8, "b") : value) + } + LLVMPositionBuilderAtEnd(b, fail_bb) + if (e.result.via_out || e.fn.result.isVoid) { + LLVMBuildRetVoid(b) + } else { + LLVMBuildRet(b, LLVMConstNull(c_return_type(e, types))) + } + return some(false) +} + + +def private emit_shutdown_trampoline(ctx : LLVMContextRef; var uids : UidNodes?; + var funcs : array) : LLVMOpaqueValue? { + var shutdown_fns <- [for (fn in funcs); fn; where fn.flags.shutdown] + if (shutdown_fns |> empty()) { + return null + } + let made = new_function(ctx, "__das_lib_ctx_shutdown", jit_fn_type($(ctx, frame : void?) : void {}), false) + let b = made.builder + defer() { + LLVMDisposeBuilder(b) + } + var ctx_arg = LLVMGetParam(made.fn, 0u) + for (fn in shutdown_fns) { + let impl_name = uids.get_dll_fn_name(fn).impl() + var impl = LLVMGetNamedFunction(g_mod, impl_name) + if (impl != null) { + LLVMBuildCall2(b, g_fn_types[impl_name], impl, array(ctx_arg), "") + } + } + LLVMBuildRetVoid(b) + return made.fn +} + + +def private emit_destroy(ctx : LLVMContextRef; var types : PrimitiveTypes?; prefix : string; + shutdown_tramp : LLVMOpaqueValue?) { + let void_ptr = types.LLVMVoidPtrType() + let made = new_function(ctx, "{prefix}_destroy", jit_fn_type($(ctx : void?) : void {}), true) + let b = made.builder + defer() { + LLVMDisposeBuilder(b) + } + var ctx_arg = LLVMGetParam(made.fn, 0u) + if (shutdown_tramp != null) { + var guard = declare_extern_fn("jit_lib_invoke_guarded", jit_fn_type($(ctx, tramp, frame : void?) : int {})) + LLVMBuildCall2(b, jit_fn_type($(ctx, tramp, frame : void?) : int {}), guard, + array(ctx_arg, LLVMBuildPointerCast(b, shutdown_tramp, void_ptr, ""), LLVMConstNull(void_ptr)), "") + } + let destroy_type = jit_fn_type($(ctx : void?) : void {}) + var destroy = declare_extern_fn("jit_destroy_standalone_ctx", destroy_type) + LLVMBuildCall2(b, destroy_type, destroy, array(ctx_arg), "") + LLVMBuildRetVoid(b) +} + + +def private emit_last_error(ctx : LLVMContextRef; var types : PrimitiveTypes?; prefix : string; + error_global : LLVMOpaqueValue?) { + let void_ptr = types.LLVMVoidPtrType() + let typ = jit_fn_type($(ctx : void?) : void? {}) + let made = new_function(ctx, "{prefix}_last_error", typ, true) + let b = made.builder + defer() { + LLVMDisposeBuilder(b) + } + var ctx_arg = LLVMGetParam(made.fn, 0u) + let from_slot = LLVMAppendBasicBlockInContext(ctx, made.fn, "from_slot") + let from_ctx = LLVMAppendBasicBlockInContext(ctx, made.fn, "from_ctx") + var no_ctx = LLVMBuildICmp(b, LLVMIntPredicate.LLVMIntEQ, ctx_arg, LLVMConstNull(void_ptr), "no_ctx") + LLVMBuildCondBr(b, no_ctx, from_slot, from_ctx) + LLVMPositionBuilderAtEnd(b, from_slot) + LLVMBuildRet(b, LLVMBuildLoad2(b, void_ptr, error_global, "create_error")) + LLVMPositionBuilderAtEnd(b, from_ctx) + LLVMBuildRet(b, LLVMBuildCall2(b, typ, declare_extern_fn("jit_lib_last_error", typ), array(ctx_arg), "")) +} + + +def private emit_shutdown_runtime(ctx : LLVMContextRef; var types : PrimitiveTypes?; prefix : string; + guard_global : LLVMOpaqueValue?) { + let made = new_function(ctx, "{prefix}_shutdown_runtime", jit_fn_type($() : void {}), true) + let b = made.builder + defer() { + LLVMDisposeBuilder(b) + } + let shutdown_type = jit_fn_type($(guard : int) : void {}) + var guard = LLVMBuildLoad2(b, types.t_int32, guard_global, "guard") + LLVMBuildCall2(b, shutdown_type, declare_extern_fn("jit_lib_shutdown", shutdown_type), array(guard), "") + LLVMBuildRetVoid(b) +} + + +def public emit_fixed_entries(ctx : LLVMContextRef; var types : PrimitiveTypes?; var uids : UidNodes?; + var funcs : array; prefix : string; + guard_global, error_global : LLVMOpaqueValue?) { + emit_destroy(ctx, types, prefix, emit_shutdown_trampoline(ctx, uids, funcs)) + emit_last_error(ctx, types, prefix, error_global) + emit_shutdown_runtime(ctx, types, prefix, guard_global) +} diff --git a/modules/dasLLVM/daslib/llvm_dll_utils.das b/modules/dasLLVM/daslib/llvm_dll_utils.das index c37d74231c..b1b01b1930 100644 --- a/modules/dasLLVM/daslib/llvm_dll_utils.das +++ b/modules/dasLLVM/daslib/llvm_dll_utils.das @@ -133,7 +133,9 @@ class public UidNodes { def private get_base(ptr : void?; hint : string) { if (!(id_map |> key_exists(ptr))) { - panic("Key not found for {hint}") + let who = thisFunc != null ? "{describe(thisFunc.at)}: {thisFunc.name}" : "a global initializer" + panic("LLVM: {who} is outside the JIT function set - it fell back to the interpreter, " + + "so its `{hint}` has no id and cannot be emitted") return DllName() } let id = unsafe(id_map[ptr]) @@ -301,17 +303,31 @@ class public DLLHandle { } -def public add_obj_extension(path : string) { - return "{path}.o" -} -def public add_dll_extension(path : string) { - return "{path}.dll" +enum public JitArtifact { + object + jit_dll + exe + shared_lib + static_lib } -def public add_exe_extension(path : string) { - return "{path}.exe" + +def public artifact_path(path : string; kind : JitArtifact) : string { + let plat = get_platform_name() + var suffix = "o" + if (kind == JitArtifact.jit_dll) { + suffix = "dll" + } elif (kind == JitArtifact.exe) { + suffix = "exe" + } elif (kind == JitArtifact.static_lib) { + suffix = plat == "windows" ? "lib" : "a" + } elif (kind == JitArtifact.shared_lib) { + suffix = plat == "windows" ? "dll" : (plat == "darwin" ? "dylib" : "so") + } + return "{path}.{suffix}" } + def public get_dll_by_path(path : string) : DLLHandle? { - return new DLLHandle(handle = load_dynamic_library(add_dll_extension(path))) + return new DLLHandle(handle = load_dynamic_library(artifact_path(path, JitArtifact.jit_dll))) } diff --git a/modules/dasLLVM/daslib/llvm_exe.das b/modules/dasLLVM/daslib/llvm_exe.das index 6bea394bc3..28b05d00d6 100644 --- a/modules/dasLLVM/daslib/llvm_exe.das +++ b/modules/dasLLVM/daslib/llvm_exe.das @@ -15,6 +15,8 @@ require llvm/daslib/llvm_jit require llvm/daslib/llvm_jit_intrin require llvm/daslib/llvm_jit_common require llvm/daslib/llvm_dll_utils +require llvm/daslib/jit_standalone +require daslib/c_api_header require daslib/ast_boost require daslib/templates_boost require daslib/macro_boost @@ -38,6 +40,7 @@ bitfield TabOperation { } +var private g_lib_registration_once = false class public CollectExternVisitor : AstVisitor { @@ -126,19 +129,19 @@ class public CollectExternVisitor : AstVisitor { registered_modules[m] = true } } else { - // Register $ and strings unless inject_main's static sweep already emitted the (non-idempotent) call. - if (!(g_exe_emitted_reg |> key_exists("jit_register_Module_BuiltIn"))) { - g_exe_emitted_reg["jit_register_Module_BuiltIn"] = true - var reg_builtin = LLVMAddFunctionWithType(g_mod, "jit_register_Module_BuiltIn", register_mod_type) - LLVMBuildCall2(ib, register_mod_type, reg_builtin, array(), "") - } - registered_modules["$"] = true - if (!(g_exe_emitted_reg |> key_exists("jit_register_Module_Strings"))) { - g_exe_emitted_reg["jit_register_Module_Strings"] = true - var reg_strings = LLVMAddFunctionWithType(g_mod, "jit_register_Module_Strings", register_mod_type) - LLVMBuildCall2(ib, register_mod_type, reg_strings, array(), "") + let once_type = jit_fn_type($(name, reg : void?) : void? {}) + var once = declare_extern_fn("jit_register_module_once", once_type) + for (pair in fixed_array(fixed_array("$", "jit_register_Module_BuiltIn"), + fixed_array("strings", "jit_register_Module_Strings"))) { + if (!(g_exe_emitted_reg |> key_exists(pair[1]))) { + g_exe_emitted_reg[pair[1]] = true + var reg_fn = declare_extern_fn(pair[1], register_mod_type) + LLVMBuildCall2(ib, once_type, once, + array(get_string_constant_ptr(ib, pair[0]), + LLVMBuildPointerCast(ib, reg_fn, g_prim_t.LLVMVoidPtrType(), "")), "") + } + registered_modules[pair[0]] = true } - registered_modules["strings"] = true } return ib } @@ -315,14 +318,23 @@ class public CollectExternVisitor : AstVisitor { if (mod_name == "ast_core" || mod_name == "ast" || mod_name == "network_core" || mod_name == "network") { needs_whole_lib = true } - // One call per thunk process-wide (not idempotent); get-or-add avoids a silent rename to an undefined symbol. return if (g_exe_emitted_reg |> key_exists(reg_fn_name)) g_exe_emitted_reg[reg_fn_name] = true var reg_fn = LLVMGetNamedFunction(g_mod, reg_fn_name) if (reg_fn == null) { reg_fn = LLVMAddFunctionWithType(g_mod, reg_fn_name, register_mod_type) } - LLVMBuildCall2(init_builder, register_mod_type, reg_fn, array(), "") + if (!g_lib_registration_once) { + LLVMBuildCall2(init_builder, register_mod_type, reg_fn, array(), "") + return + } + let once_type = jit_fn_type($(name, reg : void?) : void? {}) + var once = LLVMGetNamedFunction(g_mod, "jit_register_module_once") + if (once == null) { + once = LLVMAddFunctionWithType(g_mod, "jit_register_module_once", once_type) + } + LLVMBuildCall2(init_builder, once_type, once, + array(get_string_constant_ptr(init_builder, mod_name), reg_fn), "") } def make_call(expr : ExprCallFunc?) { @@ -673,12 +685,14 @@ def collect_external_functions(standalone_context : LLVMOpaqueValue?; ctx : LLVM } } if (!empty(extern_resolver.registered_modules) || !empty(used_modules)) { - let init_done_type = LLVMFunctionType(t.t_void, array()) + let init_done_type = LLVMFunctionType(t.t_void, array(t.t_int32)) var jit_init_done = LLVMGetNamedFunction(g_mod, "jit_initialize_modules_done") if (jit_init_done == null) { jit_init_done = LLVMAddFunctionWithType(g_mod, "jit_initialize_modules_done", init_done_type) } - LLVMBuildCall2(ib, init_done_type, jit_init_done, array(), "") + var lib_guard = LLVMGetNamedGlobal(g_mod, LIB_GUARD_GLOBAL) + var guard_arg = lib_guard == null ? t.ConstI32(0ul) : LLVMBuildLoad2(ib, t.t_int32, lib_guard, "guard") + LLVMBuildCall2(ib, init_done_type, jit_init_done, array(guard_arg), "") } // Wasm: fill per-(handled-type, field) offset globals from the TARGET runtime's annotations (now registered) — each access reads a global, not a host-baked constant. if (!empty(g_handled_field_offset_globals)) { @@ -1008,24 +1022,25 @@ def private emit_module_registration(m : Module?; dynamic_modules : table(), "") + var reg_fn = declare_extern_fn(reg_fn_name, register_mod_type) + let once_type = jit_fn_type($(name, reg : void?) : void? {}) + var once = declare_extern_fn("jit_register_module_once", once_type) + LLVMBuildCall2(builder, once_type, once, + array(get_string_constant_ptr(builder, mod_name), + reg_fn), "") } // Creates main function that initializes a standalone JIT context, registers // all compiled functions, runs init scripts, then calls the program entry point. -def public inject_main(program_context : Context?; ctx : LLVMContextRef; // nolint:STYLE037,STYLE038 — the standalone-exe entry emitter: one emission block per runtime feature, in startup order - prog : Program ?; entry_point : string; mod : LLVMOpaqueModule?; - var types : PrimitiveTypes?, var uids : UidNodes?; strict : bool; - register_all_modules : bool = false) : tuple { - g_exe_emitted_reg |> clear() // per-codegen-run: one registration call per module thunk - let builder = LLVMCreateBuilder() - defer() { - LLVMDisposeBuilder(builder) - } +struct public StandaloneFunctions { + funcs : array + used_modules : table + any_pinvoke : bool + ok : bool +} + + +def public collect_standalone_functions(prog : Program?; strict : bool) : StandaloneFunctions { // Collect ALL used functions from the program (not just JIT-compiled ones from `funcs`). // There shouldn't be any `das` functions in standalone_exe, otherwise it // will crash. @@ -1051,56 +1066,39 @@ def public inject_main(program_context : Context?; ctx : LLVMContextRef; // noli } if (strict && has_no_jit) { to_log(LOG_ERROR, "Cannot build standalone exe: some functions are no_jit in strict mode\n") - return (fn = null, link_whole_lib = false) + return <- StandaloneFunctions(funcs <- funcs, used_modules <- used_modules, any_pinvoke = any_pinvoke) } + return <- StandaloneFunctions(funcs <- funcs, used_modules <- used_modules, any_pinvoke = any_pinvoke, ok = true) +} - // --jit-register-all-modules: compiler-driver exes may recompile targets needing UnitTest at - // runtime. Force it only in main()'s dynamic-module loop below — NOT via used_modules, else - // initialize_modules() emits a second register call ("Module 'UnitTest' already created"). - var force_dynamic_modules : table - if (register_all_modules) { - force_dynamic_modules["UnitTest"] = true - } +def private find_exe_entry(var funcs : array; var uids : UidNodes?; entry_point : string) : tuple { var start_fn_name = "" var no_return = true var bool_return = false for (fn in funcs) { if (fn.name == entry_point && fn.arguments.empty()) { assume fnmna = uids.get_dll_fn_name(fn).impl() - if (fn.result.isVoid) { - start_fn_name = fnmna - no_return = true - } elif (fn.result.baseType == Type.tInt) { - start_fn_name = fnmna - no_return = false - } elif (fn.result.baseType == Type.tBool) { - start_fn_name = fnmna - no_return = false - bool_return = true - } + let is_void = fn.result.isVoid + let is_bool = fn.result.baseType == Type.tBool + let takes_it = is_void || is_bool || fn.result.baseType == Type.tInt + start_fn_name = takes_it ? fnmna : start_fn_name + no_return = takes_it ? is_void : no_return + bool_return = takes_it ? is_bool : bool_return } } - if (start_fn_name |> empty()) { - to_log(LOG_ERROR, "entrypoint `{entry_point}()` not found in input file.\n") - return (fn = null, link_whole_lib = false) - } - let main_fn_type = LLVMFunctionType(types.t_int32, - fixed_array( - types.t_int32, // argc - types.LLVMVoidPtrType() // argv - ) - ) - // wasm32-emscripten: emcc's libstandalonewasm already defines `main`, - // emit `__main_argc_argv` instead and let crt1 chain through. - let main_sym = g_target_is_wasm ? "__main_argc_argv" : "main" - let main_fn = LLVMAddFunctionWithType(mod, main_sym, main_fn_type) - let entry = LLVMAppendBasicBlockInContext(ctx, main_fn, "entry") - LLVMPositionBuilderAtEnd(builder, entry) + return (name = start_fn_name, no_return = no_return, bool_return = bool_return) +} + +def public emit_standalone_runtime_init(builder : LLVMBuilderRef; prog : Program?; var types : PrimitiveTypes?; // nolint:STYLE038 - one linear registration sequence, in startup order; a split would hide which step runs when + register_all_modules : bool; force_dynamic_modules : table; + var used_modules : table&; + var dynamic_modules : table&; + guest : bool = false) : bool { + g_exe_emitted_reg |> clear() // per-codegen-run: one registration call per module thunk // Collect dynamic module names — used by CollectExternVisitor to skip dlopen'd modules - var dynamic_modules : table for_each_registered_dynamic_module() $(_path, _mod_name, das_name) { dynamic_modules[das_name] = true } @@ -1199,30 +1197,19 @@ def public inject_main(program_context : Context?; ctx : LLVMContextRef; // noli LLVMBuildCall2(builder, init_modules_type, init_modules_fn, array(), "") // Dynamic modules load Quiet (sibling DT_NEEDED ordering): retry after the last registration - // and fatal on anything still unloadable, naming the module + dlopen error. Native only — on - // wasm those modules also register via static thunks, so the failed resolve is BY DESIGN. if (ships_dynamic && !g_target_is_wasm) { let fin_dyn_type = LLVMFunctionType(types.t_void, array()) - var fin_dyn = LLVMAddFunctionWithType(g_mod, "jit_finalize_dynamic_modules", fin_dyn_type) + let fin_dyn_name = guest ? "jit_lib_finalize_dynamic_modules" : "jit_finalize_dynamic_modules" + var fin_dyn = LLVMAddFunctionWithType(g_mod, fin_dyn_name, fin_dyn_type) LLVMBuildCall2(builder, fin_dyn_type, fin_dyn, array(), "") } - // Init argc, argv - let set_cmd_args_type = LLVMFunctionType(types.t_void, - fixed_array( - types.t_int32, // argc - types.LLVMVoidPtrType() // argv - ) - ) - var jit_set_cmd_args = LLVMAddFunctionWithType( - g_mod, "jit_set_command_line_arguments", set_cmd_args_type - ) - let argc = LLVMGetParam(main_fn, 0 |> uint); // argc - let argv = LLVMGetParam(main_fn, 1 |> uint); // argv - LLVMBuildCall2(builder, set_cmd_args_type, - jit_set_cmd_args, fixed_array(argc, argv), "") + return ships_dynamic +} +def public emit_create_standalone_ctx(builder : LLVMBuilderRef; program_context : Context?; prog : Program?; + var types : PrimitiveTypes?; any_pinvoke : bool) : LLVMOpaqueValue? { // Determine the context stack size from options (mirrors Program::getContextStackSize) var context_stack_size = uint64(prog.policies.stack) let stack_opt = find_arg(prog._options, "stack") @@ -1257,7 +1244,17 @@ def public inject_main(program_context : Context?; ctx : LLVMContextRef; // noli types.ConstI64(context_stack_size) ) let global_context = LLVMBuildCall2(builder, create_ctx_type, jit_create_context, params, "") + return global_context +} + +def public emit_standalone_context_init(builder : LLVMBuilderRef; fusion_builder : LLVMBuilderRef; // nolint:STYLE038 - one linear startup sequence sharing the context value; a split would hide which step runs when + program_context : Context ?; ctx : LLVMContextRef; prog : Program?; + mod : LLVMOpaqueModule?; var types : PrimitiveTypes?; var uids : UidNodes?; + var funcs : array; global_context : LLVMOpaqueValue?; + register_all_modules : bool; ships_dynamic : bool; + var used_modules : table&; + dynamic_modules : table) : bool { // Pre-build init_globals so its function-pointer globals exist before // collect_external_functions walks them (issue #2582). Actual init_globals(ctx) // call is emitted further down at its runtime position in main_fn. @@ -1325,9 +1322,9 @@ def public inject_main(program_context : Context?; ctx : LLVMContextRef; // noli // Whole-compiler-lib link: register the fusion engine (needed by simulate). Skip on wasm — // cross-link only sees libDaScript_runtime.a (no jit_register_fusion), and exes never read it (#2805). if (needs_whole_lib && !g_target_is_wasm) { - let reg_fusion_type = LLVMFunctionType(types.t_void, array()) - let reg_fusion_fn = LLVMAddFunctionWithType(g_mod, "jit_register_fusion", reg_fusion_type) - LLVMBuildCall2(builder, reg_fusion_type, reg_fusion_fn, array(), "") + let reg_fusion_type = jit_fn_type($() : void {}) + var reg_fusion_fn = declare_extern_fn("jit_register_fusion", reg_fusion_type) + LLVMBuildCall2(fusion_builder, reg_fusion_type, reg_fusion_fn, array(), "") } // void jit_register_standalone_variable ( Context * ctx, uint64_t index, const char * name, uint64_t mnh, uint64_t offset, int shared ) @@ -1357,10 +1354,14 @@ def public inject_main(program_context : Context?; ctx : LLVMContextRef; // noli LLVMBuildCall2(builder, init_global_var_type, init_global_var, var_params, "") } // the sealed lookups are adopted right after the context is created - before any registration - // call, so no JIT code can ever probe an unadopted table, however LLVM orders the calls { let resume_block = LLVMGetInsertBlock(builder) - LLVMPositionBuilderBefore(builder, LLVMGetNextInstruction(global_context)) + if (LLVMIsAInstruction(global_context) != null) { + LLVMPositionBuilderBefore(builder, LLVMGetNextInstruction(global_context)) + } else { + let entry = LLVMGetEntryBasicBlock(LLVMGetBasicBlockParent(resume_block)) + LLVMPositionBuilder(builder, entry, LLVMGetFirstInstruction(entry)) + } emit_exe_lookups(builder, ctx, types, global_context, collected.registered, program_context) LLVMPositionBuilderAtEnd(builder, resume_block) } @@ -1421,6 +1422,72 @@ def public inject_main(program_context : Context?; ctx : LLVMContextRef; // noli fixed_array(global_context, LLVMBuildPointerCast(builder, init_script_fn, types.LLVMVoidPtrType(), "")), "") } + return needs_whole_lib +} +def public inject_main(program_context : Context?; ctx : LLVMContextRef; // nolint:STYLE037,STYLE038 — the standalone-exe entry emitter: one emission block per runtime feature, in startup order + prog : Program ?; entry_point : string; mod : LLVMOpaqueModule?; + var types : PrimitiveTypes?, var uids : UidNodes?; strict : bool; + register_all_modules : bool = false) : tuple { + let builder = LLVMCreateBuilder() + defer() { + LLVMDisposeBuilder(builder) + } + var std_fns <- collect_standalone_functions(prog, strict) + var funcs <- std_fns.funcs + var used_modules <- std_fns.used_modules + let any_pinvoke = std_fns.any_pinvoke + if (!std_fns.ok) { + return (fn = null, link_whole_lib = false) + } + + var force_dynamic_modules : table + if (register_all_modules) { + force_dynamic_modules["UnitTest"] = true + } + + let found = find_exe_entry(funcs, uids, entry_point) + let start_fn_name = found.name + let no_return = found.no_return + let bool_return = found.bool_return + if (start_fn_name |> empty()) { + to_log(LOG_ERROR, "entrypoint `{entry_point}()` not found in input file.\n") + return (fn = null, link_whole_lib = false) + } + + let main_fn_type = LLVMFunctionType(types.t_int32, + fixed_array( + types.t_int32, // argc + types.LLVMVoidPtrType() // argv + ) + ) + let main_sym = g_target_is_wasm ? "__main_argc_argv" : "main" + let main_fn = LLVMAddFunctionWithType(mod, main_sym, main_fn_type) + let entry = LLVMAppendBasicBlockInContext(ctx, main_fn, "entry") + LLVMPositionBuilderAtEnd(builder, entry) + + var dynamic_modules : table + let ships_dynamic = emit_standalone_runtime_init(builder, prog, types, register_all_modules, + force_dynamic_modules, used_modules, dynamic_modules) + let set_cmd_args_type = LLVMFunctionType(types.t_void, + fixed_array( + types.t_int32, // argc + types.LLVMVoidPtrType() // argv + ) + ) + var jit_set_cmd_args = LLVMAddFunctionWithType( + g_mod, "jit_set_command_line_arguments", set_cmd_args_type + ) + let argc = LLVMGetParam(main_fn, 0 |> uint); // argc + let argv = LLVMGetParam(main_fn, 1 |> uint); // argv + LLVMBuildCall2(builder, set_cmd_args_type, + jit_set_cmd_args, fixed_array(argc, argv), "") + + + let global_context = emit_create_standalone_ctx(builder, program_context, prog, types, any_pinvoke) + + let needs_whole_lib = emit_standalone_context_init(builder, builder, program_context, ctx, prog, mod, + types, uids, funcs, global_context, register_all_modules, ships_dynamic, used_modules, dynamic_modules) + var main_params = fixed_array(global_context) // Match the interpreter's WebLoop: on wasm a program exporting `update` runs as init() once + // update() per browser frame + shutdown() (the browser can't block in main's while-loop). `main` // stays the desktop driver, bypassed here — so the same .das cross-compiles unchanged for web. @@ -1510,3 +1577,150 @@ def public inject_main(program_context : Context?; ctx : LLVMContextRef; // noli to_log(LOG_INFO, "LLVM JIT: standalone exe entry point generated '{start_fn_name}', {length(funcs)} functions\n") return (fn = main_fn, link_whole_lib = needs_whole_lib) } + + +let private LIB_GUARD_GLOBAL = "__das_lib_init_guard" +let private LIB_ENV_GLOBAL = "__das_lib_environment" +let private LIB_ERROR_GLOBAL = "__das_lib_create_error" +let private LIB_RUNTIME_INIT = "__das_lib_runtime_init" +let private LIB_CTX_INIT = "__das_lib_ctx_init" +var public g_lib_exports : array + + +def private emit_lib_globals(var types : PrimitiveTypes?) { + var guard = LLVMAddGlobal(g_mod, types.t_int32, LIB_GUARD_GLOBAL) + set_private_linkage(guard) + LLVMSetInitializer(guard, types.ConstI32(0ul)) + for (name in [LIB_ENV_GLOBAL, LIB_ERROR_GLOBAL]) { + var slot = LLVMAddGlobal(g_mod, types.LLVMVoidPtrType(), name) + set_private_linkage(slot) + LLVMSetInitializer(slot, LLVMConstNull(types.LLVMVoidPtrType())) + } +} + + +def private emit_runtime_init(ctx : LLVMContextRef; prog : Program?; var types : PrimitiveTypes?; + register_all_modules : bool; var used_modules : table&; + var dynamic_modules : table&) : tuple { + emit_lib_globals(types) + let void_fn_type = jit_fn_type($() : void {}) + let made = new_function(ctx, LIB_RUNTIME_INIT, void_fn_type, false) + let fresh_init_modules = LLVMGetNamedFunction(g_mod, "initialize_modules") == null + var init_modules = declare_extern_fn("initialize_modules", void_fn_type) + if (fresh_init_modules) { + set_private_linkage(init_modules) + } + var force_dynamic_modules : table + if (register_all_modules) { + force_dynamic_modules["UnitTest"] = true + } + let ships = emit_standalone_runtime_init(made.builder, prog, types, register_all_modules, + force_dynamic_modules, used_modules, dynamic_modules, true) + return (fn = made.fn, builder = made.builder, ships_dynamic = ships) +} + + +def private emit_create(ctx : LLVMContextRef; program_context : Context?; prog : Program?; mod : LLVMOpaqueModule?; + var types : PrimitiveTypes?; var uids : UidNodes?; prefix : string; + var sf : StandaloneFunctions; runtime_init : LLVMOpaqueValue?; + runtime_builder : LLVMBuilderRef; register_all_modules : bool; ships_dynamic : bool; + dynamic_modules : table) : tuple { + let void_ptr = types.LLVMVoidPtrType() + var guard_global = LLVMGetNamedGlobal(g_mod, LIB_GUARD_GLOBAL) + var env_global = LLVMGetNamedGlobal(g_mod, LIB_ENV_GLOBAL) + var error_global = LLVMGetNamedGlobal(g_mod, LIB_ERROR_GLOBAL) + + let ctx_init = new_function(ctx, LIB_CTX_INIT, jit_fn_type($(ctx, frame : void?) : void {}), false) + let made = new_function(ctx, "{prefix}_create", jit_fn_type($() : void? {}), true) + let b = made.builder + defer() { + LLVMDisposeBuilder(b) + LLVMDisposeBuilder(ctx_init.builder) + } + let run_once_type = jit_fn_type($(guard, env, init : void?) : int {}) + var run_once = declare_extern_fn("jit_lib_run_once", run_once_type) + var registered = LLVMBuildCall2(b, run_once_type, run_once, + array(LLVMBuildPointerCast(b, guard_global, void_ptr, ""), + LLVMBuildPointerCast(b, env_global, void_ptr, ""), + LLVMBuildPointerCast(b, runtime_init, void_ptr, "")), "registered") + let ready_bb = LLVMAppendBasicBlockInContext(ctx, made.fn, "ready") + let declined_bb = LLVMAppendBasicBlockInContext(ctx, made.fn, "declined") + var can_run = LLVMBuildICmp(b, LLVMIntPredicate.LLVMIntNE, registered, types.ConstI32(0ul), "can_run") + LLVMBuildCondBr(b, can_run, ready_bb, declined_bb) + LLVMPositionBuilderAtEnd(b, declined_bb) + LLVMBuildRet(b, LLVMConstNull(void_ptr)) + LLVMPositionBuilderAtEnd(b, ready_bb) + var global_context = emit_create_standalone_ctx(b, program_context, prog, types, sf.any_pinvoke) + let guard_type = jit_fn_type($(ctx, tramp, frame : void?) : int {}) + var guard = declare_extern_fn("jit_lib_invoke_guarded", guard_type) + var ok = LLVMBuildCall2(b, guard_type, guard, + array(global_context, LLVMBuildPointerCast(b, ctx_init.fn, void_ptr, ""), LLVMConstNull(void_ptr)), "ok") + let finish_type = jit_fn_type($(ctx : void?; ok : int; err : void?) : void? {}) + var finish = declare_extern_fn("jit_lib_create_finish", finish_type) + LLVMBuildRet(b, LLVMBuildCall2(b, finish_type, finish, + array(global_context, ok, LLVMBuildPointerCast(b, error_global, void_ptr, "")), "")) + + var ctx_param = LLVMGetParam(ctx_init.fn, 0u) + let whole = emit_standalone_context_init(ctx_init.builder, runtime_builder, program_context, ctx, + prog, mod, types, uids, sf.funcs, ctx_param, register_all_modules, ships_dynamic, + sf.used_modules, dynamic_modules) + LLVMBuildRetVoid(ctx_init.builder) + return (fn = made.fn, link_whole_lib = whole) +} + + +[arch(at="../ARCHITECTURE_LIB.md#lib-runtime-scope")] +def public inject_lib(program_context : Context?; ctx : LLVMContextRef; prog : Program?; + mod : LLVMOpaqueModule?; var types : PrimitiveTypes?; var uids : UidNodes?; + strict : bool; export_all : bool; output_path : string; + register_all_modules : bool = false) : tuple { + g_lib_registration_once = true + defer() { + g_lib_registration_once = false + } + g_lib_exports |> clear() + let prefix = lib_prefix_from_path(output_path) + if (prefix |> empty()) { + to_log(LOG_ERROR, "LLVM LIB: cannot derive a C symbol prefix from the output path `{output_path}` - name it after a C identifier\n") + return (fn = null, link_whole_lib = false) + } + let names = CNames(prefix = prefix, this_module = prog.getThisModule) + var selected <- collect_c_exports(prog, names, export_all) + for (r in selected.errors) { + failed(r.message) + } + if (!(selected.errors |> empty())) { + return (fn = null, link_whole_lib = false) + } + var exports <- selected.exports + if (exports |> empty()) { + to_log(LOG_ERROR, "LLVM LIB: nothing to export - annotate functions with [export_c], or pass -lib-export-all\n") + return (fn = null, link_whole_lib = false) + } + var sf <- collect_standalone_functions(prog, strict) + if (!sf.ok) { + return (fn = null, link_whole_lib = false) + } + var dynamic_modules : table + let rt = emit_runtime_init(ctx, prog, types, register_all_modules, sf.used_modules, dynamic_modules) + let created = emit_create(ctx, program_context, prog, mod, types, uids, prefix, sf, rt.fn, + rt.builder, register_all_modules, rt.ships_dynamic, dynamic_modules) + LLVMBuildRetVoid(rt.builder) + LLVMDisposeBuilder(rt.builder) + emit_fixed_entries(ctx, types, uids, sf.funcs, prefix, + LLVMGetNamedGlobal(g_mod, LIB_GUARD_GLOBAL), LLVMGetNamedGlobal(g_mod, LIB_ERROR_GLOBAL)) + var emitted : array + for (e in exports) { + let thunk = emit_thunk(ctx, e, uids, types, export_all) + if (thunk |> is_none()) { + return (fn = null, link_whole_lib = false) + } + if (!(thunk |> unwrap())) { + emitted |> push_clone(e) + } + } + to_log(LOG_INFO, "LLVM LIB: C entry points generated: {prefix}_create/destroy/last_error plus {length(emitted)} exports, {length(sf.funcs)} functions\n") + g_lib_exports |> clear() + g_lib_exports |> push_clone_from(emitted) + return (fn = created.fn, link_whole_lib = created.link_whole_lib) +} diff --git a/modules/dasLLVM/daslib/llvm_jit.das b/modules/dasLLVM/daslib/llvm_jit.das index 38cf6f943b..5a5a8c7e90 100644 --- a/modules/dasLLVM/daslib/llvm_jit.das +++ b/modules/dasLLVM/daslib/llvm_jit.das @@ -91,7 +91,7 @@ def set_debug_linkage(value : LLVMOpaqueValue?) { LLVMSetLinkage(value, LLVMLinkage.LLVMInternalLinkage) } -def set_public_linkage(value : LLVMOpaqueValue?) { +def public set_public_linkage(value : LLVMOpaqueValue?) { LLVMSetLinkage(value, LLVMLinkage.LLVMDLLExportLinkage) LLVMSetDLLStorageClass(value, LLVMDLLStorageClass.LLVMDLLExportStorageClass) } @@ -8156,5 +8156,7 @@ def init_llvm_jit_module_options() { this_module() |> add_module_option("jit_target", Type.tString) this_module() |> add_module_option("jit_split_modules", Type.tInt) this_module() |> add_module_option("jit_obj_cache", Type.tBool) + this_module() |> add_module_option("jit_lib", Type.tBool) + this_module() |> add_module_option("jit_lib_export_marked", Type.tBool) } } diff --git a/modules/dasLLVM/daslib/llvm_jit_cli.das b/modules/dasLLVM/daslib/llvm_jit_cli.das index 35b7eaf975..cbcef25d2d 100644 --- a/modules/dasLLVM/daslib/llvm_jit_cli.das +++ b/modules/dasLLVM/daslib/llvm_jit_cli.das @@ -74,6 +74,22 @@ struct public JitCliOptions { @clarg_doc = "JIT: explicit linker binary (overrides default c++/clang/lld-link). Windows-MSVC builds a link.exe-flavored cmd (/DLL /OUT:) — pass a link.exe-compatible linker; elsewhere match the compiler daslang was built with to avoid sanitizer runtime mismatch." path_to_linker : Option + @clarg_name = "jit-lib" + @clarg_doc = "JIT: emit a C-ABI native library plus its C header instead of running the program. Script-level pin: options jit_lib = true" + lib : Option + + @clarg_name = "jit-lib-export-marked" + @clarg_doc = "JIT (-lib): export every function the program already marks [export], not only the [export_c] ones. Selection only - unlike -lib-export-all it marks nothing itself, so a non-representable signature is still a hard error. Script-level pin: options jit_lib_export_marked = true" + lib_export_marked : Option + + @clarg_name = "jit-lib-static" + @clarg_doc = "JIT (-lib): produce a static archive (.a / .lib) instead of a shared library; the host then links libDaScript_runtime itself (the generated header records what). Under this flag --jit-path-to-linker names the ARCHIVER (llvm-ar / ar / lib), not a linker" + lib_static : Option + + @clarg_name = "jit-lib-bindings" + @clarg_doc = "JIT (-lib): also write daslang `[extern]` bindings for the emitted library to this path, so a daslang host requires them instead of hand-writing the declarations or parsing the C header" + lib_bindings : Option + @clarg_name = "jit-register-all-modules" @clarg_doc = "JIT (-exe only): register all builtin native modules (math, fio, dasbind, ...) at exe startup, so a standalone compiler-driver exe can recompile arbitrary daslang at runtime" register_all_modules : Option diff --git a/modules/dasLLVM/daslib/llvm_jit_common.das b/modules/dasLLVM/daslib/llvm_jit_common.das index ad97f866aa..7aa4a31383 100644 --- a/modules/dasLLVM/daslib/llvm_jit_common.das +++ b/modules/dasLLVM/daslib/llvm_jit_common.das @@ -537,6 +537,15 @@ def public jit_extern_type(t : Type) : LLVMTypeRef { return t == Type.tVoid ? g_prim_t.t_void : base_type_to_llvm_type(t) } +def public declare_extern_fn(name : string; typ : LLVMTypeRef) : LLVMOpaqueValue? { + var fn = LLVMGetNamedFunction(g_mod, name) + if (fn == null) { + fn = LLVMAddFunctionWithType(g_mod, name, typ) + } + return fn +} + + def public jit_add_extern(name : string; typ : LLVMTypeRef; fnAddr : void?; attrs : LLVMOpaqueAttributeRef? []) { var f = LLVMAddFunctionWithType(g_mod, name, typ) LLVMAddGlobalMapping(g_engine, f, fnAddr) @@ -1011,7 +1020,8 @@ def public with_default_target_machine(opt_level : uint; use_host_cpu : bool; // Offline AOT: emit a native object only, no linker step (jit_emit_object path). // use_host_cpu=false (default) targets a generic CPU so the .o is portable; true bakes host features. -def public emit_object_only(mod : LLVMOpaqueModule?; out_path : string; use_host_cpu : bool = false) { +def public emit_object_only(mod : LLVMOpaqueModule?; out_path : string; use_host_cpu : bool = false) : bool { + var ok = true // Codegen level 3 deliberate: shipped artifacts, no cache guard (ARCHITECTURE.md 1.2) with_default_target_machine(3u, use_host_cpu) $(targetMachine : LLVMTargetMachineRef) { // A module with no triple and no layout leaves the object to whatever the backend @@ -1021,13 +1031,15 @@ def public emit_object_only(mod : LLVMOpaqueModule?; out_path : string; use_host LLVMSetTarget(mod, LLVMGetDefaultTargetTriple()) LLVMDisposeTargetData(dl) let error : string? - let file = add_obj_extension(out_path) + let file = artifact_path(out_path, JitArtifact.object) let filetype = LLVMCodeGenFileType.LLVMObjectFile let failed = LLVMTargetMachineEmitToFile(targetMachine, mod, file, filetype, error) if (failed != 0) { - panic("emit_object_only: LLVMTargetMachineEmitToFile failed for {file}") + to_log(LOG_ERROR, "emit_object_only: LLVMTargetMachineEmitToFile failed for {file}\n") + ok = false } } + return ok } // nolint:STYLE014 @@ -1059,7 +1071,7 @@ def public link_dll_from_objects(objs : array; out_path : string; path_t return false } let phase_tm = ref_time_ticks() - let ok = create_shared_library(objs[0], add_dll_extension(out_path), "{path_to_dascript_lib}", "{path_to_linker}", "\"@{rsp}\" {linker_string}", true, link_whole_lib, debug_info) + let ok = create_shared_library(objs[0], artifact_path(out_path, JitArtifact.jit_dll), "{path_to_dascript_lib}", "{path_to_linker}", "\"@{rsp}\" {linker_string}", true, link_whole_lib, debug_info) if (log_time) { to_log(LOG_INFO, "LLVM JIT time: link {jit_sec(get_time_usec(phase_tm))} ({length(objs)} objects)\n") } @@ -1069,44 +1081,41 @@ def public link_dll_from_objects(objs : array; out_path : string; path_t // @out_path - folder + file name, without extension. @path_to_dascript_lib - required on Windows, // no effect on Linux. @path_to_linker - linker override; Windows-MSVC defaults to lld-link (link.exe // flavor) from the LLVM package, c++/clang elsewhere. -def public write_dll(mod : LLVMOpaqueModule?; out_path : string; path_to_dascript_lib, path_to_linker, linker_string : string; link_whole_lib : bool; debug_info : bool = false; log_time : bool = false; codegen_opt_level : uint = 3u) { - // JIT DLL cache: emitted artifact only ever runs on this host. - with_default_target_machine(codegen_opt_level, true) $(targetMachine : LLVMTargetMachineRef) { +def public write_artifact(mod : LLVMOpaqueModule?; out_path : string; kind : JitArtifact; + path_to_dascript_lib, path_to_linker, linker_string : string; + link_whole_lib : bool; use_host_cpu : bool; debug_info : bool = false; + log_time : bool = false; codegen_opt_level : uint = 3u) : bool { + var emitted = true + with_default_target_machine(codegen_opt_level, kind == JitArtifact.jit_dll ? true : use_host_cpu) $(targetMachine : LLVMTargetMachineRef) { let error : string? - let file = add_obj_extension(out_path) - let filetype = LLVMCodeGenFileType.LLVMObjectFile + let file = artifact_path(out_path, JitArtifact.object) var phase_tm = ref_time_ticks() - let failed = LLVMTargetMachineEmitToFile(targetMachine, mod, file, filetype, error) - if (failed != 0) { - panic("write_dll: LLVMTargetMachineEmitToFile failed for {file}") + if (LLVMTargetMachineEmitToFile(targetMachine, mod, file, LLVMCodeGenFileType.LLVMObjectFile, error) != 0) { + to_log(LOG_ERROR, "write_artifact: LLVMTargetMachineEmitToFile failed for {file}\n") + emitted = false + return } let t_emit = get_time_usec(phase_tm) phase_tm = ref_time_ticks() - let ok = create_shared_library(file, add_dll_extension(out_path), "{path_to_dascript_lib}", "{path_to_linker}", "{linker_string}", true, link_whole_lib, debug_info) + let target = artifact_path(out_path, kind) + var ok = false + if (kind == JitArtifact.static_lib) { + ok = create_static_library(file, target, "{path_to_linker}") + } else { + ok = create_shared_library(file, target, "{path_to_dascript_lib}", "{path_to_linker}", + "{linker_string}", kind != JitArtifact.exe, link_whole_lib, debug_info) + } if (log_time) { to_log(LOG_INFO, "LLVM JIT time: emit-obj {jit_sec(t_emit)} link {jit_sec(get_time_usec(phase_tm))}\n") } if (!ok) { - panic("write_dll: link failed for {out_path}") + to_log(LOG_ERROR, "write_artifact: {kind == JitArtifact.static_lib ? "archive" : "link"} failed for {target}\n") + emitted = false } } + return emitted } -def public write_exe(mod : LLVMOpaqueModule?; out_path : string; path_to_dascript_lib, path_to_linker, linker_string : string; link_whole_lib : bool; use_host_cpu : bool; debug_info : bool = false) { - // Standalone exe: generic (redistributable) by default; use_host_cpu targets the box so - // tuner-generated host-specific IR legalizes (the generic target aborts codegen on it). - // Codegen level 3 deliberate: shipped artifacts, no cache guard (ARCHITECTURE.md 1.2) - with_default_target_machine(3u, use_host_cpu) $(targetMachine : LLVMTargetMachineRef) { - let error : string? - let file = add_obj_extension(out_path) - let filetype = LLVMCodeGenFileType.LLVMObjectFile - LLVMTargetMachineEmitToFile(targetMachine, mod, file, filetype, error) - let ok = create_shared_library(file, add_exe_extension(out_path), "{path_to_dascript_lib}", "{path_to_linker}", "{linker_string}", false, link_whole_lib, debug_info) - if (!ok) { - panic("write_exe: link failed for {out_path}") - } - } -} // Locate the wasm64 (memory64) libDaScript_runtime.a. Non-empty `override_path` wins (e.g. // --jit-runtime-lib CLI flag; a missing file logs a warning); otherwise auto-locate at @@ -1124,7 +1133,8 @@ def private find_runtime_lib(override_path : string) : Option { // Cross-compile a module to wasm64 (memory64) and link a runnable .wasm via emcc against the // web/output64 runtime archive (-sMEMORY64=1). triple defaults to wasm64-unknown-emscripten; // needs_runtime gates linking libDaScript_runtime.a (true=link if found else warn; false=omit). -def public write_wasm(mod : LLVMOpaqueModule?; out_path : string; triple : string; path_to_emcc : string; needs_runtime : bool; explicit_runtime : string = ""; emit_object_only : bool = false) { +def public write_wasm(mod : LLVMOpaqueModule?; out_path : string; triple : string; path_to_emcc : string; needs_runtime : bool; explicit_runtime : string = ""; emit_object_only : bool = false) : bool { + var wasm_ok = true LLVMInitializeWasmTarget() let real_triple = (!(triple |> empty()) ? triple : "wasm64-unknown-emscripten") let runtime_lib = find_runtime_lib(explicit_runtime) @@ -1147,14 +1157,16 @@ def public write_wasm(mod : LLVMOpaqueModule?; out_path : string; triple : strin let error : string? // emit_object_only: write the object verbatim to out_path (no `.o.o` // double extension) so the caller has a predictable, linkable artifact. - let obj = emit_object_only ? out_path : add_obj_extension(out_path) + let obj = emit_object_only ? out_path : artifact_path(out_path, JitArtifact.object) let filetype = LLVMCodeGenFileType.LLVMObjectFile LLVMTargetMachineEmitToFile(targetMachine, mod, obj, filetype, error) if (emit_object_only) { // The caller (e.g. daspkg release wasm) runs emcc; verify the object // exists so a failed emit doesn't silently hand off a missing file. if (error != null || !stat(obj).is_valid) { - panic("write_wasm: failed to emit object {obj} (LLVMTargetMachineEmitToFile error)") + to_log(LOG_ERROR, "write_wasm: failed to emit object {obj} (LLVMTargetMachineEmitToFile error)\n") + wasm_ok = false + return } to_log(LOG_INFO, "wasm: emitted object {obj} (link skipped; --jit-emit-object)\n") return @@ -1162,9 +1174,11 @@ def public write_wasm(mod : LLVMOpaqueModule?; out_path : string; triple : strin let runtime_arg = needs_runtime ? (runtime_lib ?? "") : "" let ok = link_wasm(obj, "{out_path}.wasm", runtime_arg, path_to_emcc, true) if (!ok) { - panic("write_wasm: link failed for {out_path}.wasm") + to_log(LOG_ERROR, "write_wasm: link failed for {out_path}.wasm\n") + wasm_ok = false } } + return wasm_ok } def public build_string_constant(message : string) { diff --git a/modules/dasLLVM/daslib/llvm_jit_link.das b/modules/dasLLVM/daslib/llvm_jit_link.das index af7994fc96..9ff1aeea65 100644 --- a/modules/dasLLVM/daslib/llvm_jit_link.das +++ b/modules/dasLLVM/daslib/llvm_jit_link.das @@ -67,7 +67,7 @@ def public run_jit_linked(prog : Program?; var ctx : Context?) : bool { if (!plan.use_dll) { return run_jit_in_emitter(prog, ctx, plan.log_jit_time, t_hash, 0) } - if (empty(plan.candidates)) { + if (empty(plan.candidates) && !plan.gen_lib) { to_log(LOG_INFO, "LLVM JIT: 0 functions to jit ({length(plan.disabled)} marked no_jit) - the WHOLE program runs interpreted\n") return true } diff --git a/modules/dasLLVM/daslib/llvm_jit_plan.das b/modules/dasLLVM/daslib/llvm_jit_plan.das index b52fba518d..69474bad1e 100644 --- a/modules/dasLLVM/daslib/llvm_jit_plan.das +++ b/modules/dasLLVM/daslib/llvm_jit_plan.das @@ -719,6 +719,10 @@ struct public JitPlan { debug_info : bool dump_ir : bool gen_exe : bool + gen_lib : bool + lib_static : bool + lib_export_marked : bool + lib_bindings : string exe_main : string compile_only : bool emit_aot_object : bool @@ -774,7 +778,19 @@ def public make_jit_plan(prog : Program?; var ctx : Context?; announce : bool) : to_log(LOG_WARNING, "LLVM JIT: dll mode requested but this daslang build is static - no DLL cache, " + "every run pays full in-memory codegen\n") } - plan.gen_exe = prog.policies.jit_exe_mode + plan.gen_lib = cli_opts.lib |> unwrap_or((prog._options |> find_arg("jit_lib")) ?as tBool ?? false) + plan.gen_exe = prog.policies.jit_exe_mode || plan.gen_lib + plan.lib_static = cli_opts.lib_static |> unwrap_or(false) + plan.lib_bindings = cli_opts.lib_bindings |> unwrap_or("") + plan.lib_export_marked = cli_opts.lib_export_marked |> unwrap_or((prog._options |> find_arg("jit_lib_export_marked")) ?as tBool ?? false) + if (announce && !plan.gen_lib) { + if (cli_opts.lib_static |> is_some()) { + to_log(LOG_WARNING, "LLVM JIT: --jit-lib-static applies to -lib only - ignored for this run\n") + } + if (cli_opts.lib_bindings |> is_some()) { + to_log(LOG_WARNING, "LLVM JIT: --jit-lib-bindings applies to -lib only - ignored for this run\n") + } + } plan.exe_main = "main" // --jit-compile-only: build + verify + optimize (+dump), write/load/install nothing — // the program runs interpreted. Also bypasses the DLL cache probe, so the diff --git a/modules/dasLLVM/daslib/llvm_jit_run.das b/modules/dasLLVM/daslib/llvm_jit_run.das index 5e32460b1c..942e16db08 100644 --- a/modules/dasLLVM/daslib/llvm_jit_run.das +++ b/modules/dasLLVM/daslib/llvm_jit_run.das @@ -8,6 +8,8 @@ require llvm/daslib/llvm_boost require llvm/daslib/llvm_dll_utils require llvm/daslib/llvm_jit_plan public require llvm/daslib/llvm_exe +require llvm/daslib/jit_standalone +require daslib/c_api_header require llvm/daslib/llvm_aot require llvm/daslib/llvm_jit require llvm/daslib/llvm_jit_common @@ -35,7 +37,7 @@ var LINK_WHOLE_LIB = false // when true, standalone exe links against the whole // Read by tests-cpp/small/test_jit_emitter_pin.cpp: FNV-1a64 of the emitter sources // (normalized to LF; file list in the test) -let LLVM_JIT_EMITTER_HASH : uint64 = 0xa2ab0760ed831369ul +let LLVM_JIT_EMITTER_HASH : uint64 = 0x6278c66628ebdb61ul def private apply_fast_math_to_module(m : LLVMOpaqueModule?) { var fn = LLVMGetFirstFunction(m) @@ -310,8 +312,8 @@ def private run_split_codegen(prog : Program?; ctx : Context?; funcs : array push(obj) // link order = partition order, hit or miss if (obj_cache && stat(obj).is_valid) { cached++ @@ -448,6 +450,9 @@ def public run_jit(prog : Program?; var ctx : Context?) : bool { // nolint:STYL assume debug_info = plan.debug_info assume dump_ir = plan.dump_ir assume gen_exe = plan.gen_exe + assume gen_lib = plan.gen_lib + assume lib_static = plan.lib_static + assume lib_bindings = plan.lib_bindings assume exe_main = plan.exe_main assume compile_only = plan.compile_only assume emit_aot_object = plan.emit_aot_object @@ -493,6 +498,8 @@ def public run_jit(prog : Program?; var ctx : Context?) : bool { // nolint:STYL init_jit_target_flags(target_triple, use_host_cpu) var funcs : array var visitorDisabled = 0 + var emit_failed = false + defer() { reset_codegen_accumulators(); } var disableJitVisitor = new DisableJitVisitor() make_visitor(*disableJitVisitor) $(disableJitVisitorAdapter) { for (fun in plan.candidates) { @@ -515,7 +522,7 @@ def public run_jit(prog : Program?; var ctx : Context?) : bool { // nolint:STYL // emit_aot_object always writes an object even with no functions to emit (a fully-no_aot // module): an empty .o with just the fileinfo ctor, so the build always has the file it // expects and those functions interpret at load (as Program::linkCppAot skips them). - if (!empty(funcs) || emit_aot_object) { + if (!empty(funcs) || emit_aot_object || gen_lib) { // the plan's fold over the candidates - the first aot-hash per function lands there (cached after) let t_hash = get_time_usec(phase_tm) phase_tm = ref_time_ticks() @@ -600,7 +607,18 @@ def public run_jit(prog : Program?; var ctx : Context?) : bool { // nolint:STYL reopen_and_gc_dll(g_dynamic_lib_handle, output_path, funcs, disabled, uids, lto_probe ? "" : dll_hash_basename, sres.objs) } elif (recompile_prog) { jit_has_externals = irgen_functions(ctx, uids, attrs, funcs, jit_flags, jit_mode, prog) - if (gen_exe) { + if (gen_lib) { + let export_marked = plan.lib_export_marked + let res = inject_lib(ctx, g_ctx, prog, g_mod, g_prim_t, uids, exe_strict, + prog.policies.export_public_functions || export_marked, output_path, + plan.register_all_modules) + LINK_WHOLE_LIB = res.link_whole_lib + if (res.fn == null) { + finalize_jit(use_dll, gen_exe, g_dynamic_lib_handle) + to_log(LOG_ERROR, "LLVM LIB: no library written for {output_path}\n") + return false + } + } elif (gen_exe) { let res = inject_main(ctx, g_ctx, prog, exe_main, g_mod, g_prim_t, uids, exe_strict, plan.register_all_modules) LINK_WHOLE_LIB = res.link_whole_lib @@ -645,19 +663,36 @@ def public run_jit(prog : Program?; var ctx : Context?) : bool { // nolint:STYL if (compile_only) { to_log(LOG_INFO, "LLVM JIT: compile-only - module built, optimized and verified; no artifact written, nothing installed\n") } elif (emit_aot_object) { - emit_object_only(g_mod, output_path, use_host_cpu) + emit_failed ||= !emit_object_only(g_mod, output_path, use_host_cpu) + } elif (gen_lib) { + mkdir_rec(dir_name(output_path)) + emit_failed ||= !write_artifact(g_mod, output_path, lib_static ? JitArtifact.static_lib : JitArtifact.shared_lib, + path_to_shared_lib, path_to_linker, linker_string, LINK_WHOLE_LIB, exe_host_cpu, + debug_info, log_jit_time) + let lib_names = CNames(prefix = lib_prefix_from_path(output_path), this_module = prog.getThisModule) + if (!emit_c_header(g_lib_exports, lib_names, string(prog.getThisModule.fileName), + output_path, lib_link_note(lib_static, LINK_WHOLE_LIB))) { + to_log(LOG_ERROR, "LLVM LIB: the library is written but its C header is not - {output_path}.h\n") + emit_failed = true + } + if (!(lib_bindings |> empty())) { + emit_failed ||= !write_das_bindings(g_lib_exports, lib_names, string(prog.getThisModule.fileName), + output_path, lib_bindings, lib_static) + } } elif (gen_wasm) { mkdir_rec(dir_name(output_path)) // path_to_linker is reused as the optional emcc override // for the wasm link (mirrors how it's used for the host // linker, lld-link/clang/c++, on the host path). - write_wasm(g_mod, output_path, target_triple, path_to_linker, jit_has_externals, runtime_lib_override, emit_object) + emit_failed ||= !write_wasm(g_mod, output_path, target_triple, path_to_linker, jit_has_externals, runtime_lib_override, emit_object) } elif (gen_exe) { mkdir_rec(dir_name(output_path)) - write_exe(g_mod, output_path, path_to_shared_lib, path_to_linker, linker_string, LINK_WHOLE_LIB, exe_host_cpu, debug_info) + emit_failed ||= !write_artifact(g_mod, output_path, JitArtifact.exe, path_to_shared_lib, path_to_linker, + linker_string, LINK_WHOLE_LIB, exe_host_cpu, debug_info) } elif (use_dll) { mkdir_rec(dir_name(output_path)) - write_dll(g_mod, output_path, path_to_shared_lib, path_to_linker, linker_string, LINK_WHOLE_LIB, debug_info, log_jit_time, opt_level |> uint) + emit_failed ||= !write_artifact(g_mod, output_path, JitArtifact.jit_dll, path_to_shared_lib, path_to_linker, + linker_string, LINK_WHOLE_LIB, true, debug_info, log_jit_time, opt_level |> uint) let no_keep_objs : array // monolith artifacts all carry the dll-hash prefix reopen_and_gc_dll(g_dynamic_lib_handle, output_path, funcs, disabled, uids, dll_hash_basename, no_keep_objs) } @@ -678,7 +713,7 @@ def public run_jit(prog : Program?; var ctx : Context?) : bool { // nolint:STYL } jit_log_report(funcs, true, opt_level, true, false, 0, debug_info, log_jit_time, totalTime, t_hash, t_init, t_declare, t_probe, t_irgen, t_opt, t_emit, 0, 0) - return true + return !emit_failed } if (use_dll) { assert(g_dynamic_lib_handle.handle != null) @@ -721,5 +756,5 @@ def public run_jit(prog : Program?; var ctx : Context?) : bool { // nolint:STYL } else { to_log(LOG_INFO, "LLVM JIT: 0 functions to jit ({length(disabled)} marked no_jit, {visitorDisabled} disabled by content) - the WHOLE program runs interpreted\n") } - return true + return !emit_failed } diff --git a/skills/daslang/references/cli-and-config.md b/skills/daslang/references/cli-and-config.md index 56e6ed1502..5b4b9d2253 100644 --- a/skills/daslang/references/cli-and-config.md +++ b/skills/daslang/references/cli-and-config.md @@ -83,6 +83,9 @@ interpreter wires it to `-?` instead: help : bool ``` +A `-lib` library never owns argv at all: the host's process arguments are whatever the host was +started with, so a library reads its configuration from its C parameters, not from `clargs`. + Only an `-exe` binary owns argv, so `-h` / `--help` - and `parse_args_with_help`'s automatic help flag - are reachable only there. diff --git a/skills/daslang/references/modules-and-stdlib.md b/skills/daslang/references/modules-and-stdlib.md index 9fc4a86305..cf231e5a73 100644 --- a/skills/daslang/references/modules-and-stdlib.md +++ b/skills/daslang/references/modules-and-stdlib.md @@ -23,9 +23,9 @@ The only enforced ordering is `module` before any type declaration; `options` / `require` otherwise interleave. A file with no `module` line is a program, named by its file stem. `[export]` makes a function callable from the host by name, and `[export_c]` is an `[export]` that -also crosses to C - a standalone context declares it in the C half of the header it generates, so a -C host needs no daslang API (`require daslib/export_c`; `[export_c(name = "...")]` picks the C -symbol, which is how two overloads both reach C); `[init]` / `[finalize]` run at context +`daslang -lib` also surfaces as a C function in the header it generates (`-lib`/`-jit`/`-exe` carry +the annotation; a non-JIT compile of that source needs `require daslib/export_c`) (`[export_c(name = "...")]` +picks the C symbol, which is how two overloads both reach C); `[init]` / `[finalize]` run at context init / shutdown (no arguments, no return). `main` is a convention, not a keyword: it returns `void` unless declared `def main() : int`, whose return value is the process exit code (do not `panic` to force one). diff --git a/src/ast/ast_export.cpp b/src/ast/ast_export.cpp index 27022f2a71..aa9801ab52 100644 --- a/src/ast/ast_export.cpp +++ b/src/ast/ast_export.cpp @@ -132,6 +132,14 @@ namespace das { return true; }, "*"); } + void exportPublicFunctions( Module * thisModule ) { + for ( auto & fn : thisModule->functions.each() ) { + if ( fn->privateFunction || fn->builtIn || fn->generated || fn->isTemplate ) continue; + if ( fn->macroInit || fn->macroFunction || fn->init || fn->shutdown ) continue; + if ( fn->isClassMethod || fn->lambda || fn->generator || fn->fromGeneric ) continue; + fn->exports = true; + } + } void markModuleVarsUsed( ModuleLibrary &, Module * inWhichModule ) { for ( auto & var : inWhichModule->globals.each() ) { program->setUsed(var, false); @@ -346,6 +354,7 @@ namespace das { MarkSymbolUse vis(this, false); vis.tw = logs; visit(vis); + if ( policies.export_public_functions ) vis.exportPublicFunctions(thisModule.get()); vis.markUsedFunctions(library, false, false, nullptr); vis.markVarsUsed(library, false); } diff --git a/src/ast/ast_module.cpp b/src/ast/ast_module.cpp index 1a4ab1ffb8..7702b7cb49 100644 --- a/src/ast/ast_module.cpp +++ b/src/ast/ast_module.cpp @@ -127,9 +127,6 @@ namespace das { atomic g_envTotal(0); - // from module_builtin_fio.cpp — modules whose .shared_module dlopen failed (Quiet) - DAS_API string describe_pending_dynamic_modules(); - static void daslang_atexit_audit() { int n = g_envTotal.load(); if ( n != 0 ) { diff --git a/src/ast/ast_parse.cpp b/src/ast/ast_parse.cpp index 1a4cbd45a4..1207bb941b 100644 --- a/src/ast/ast_parse.cpp +++ b/src/ast/ast_parse.cpp @@ -1663,9 +1663,6 @@ namespace das { } } - // from module_builtin_fio.cpp — modules whose .shared_module dlopen failed (Quiet) - DAS_API string describe_pending_dynamic_modules(); - ProgramPtr reportPrerequisitesErrors ( const string & fileName, const vector & missing, diff --git a/src/builtin/jit_runtime.cpp b/src/builtin/jit_runtime.cpp index 1f131cc534..ba5fa7cd7d 100644 --- a/src/builtin/jit_runtime.cpp +++ b/src/builtin/jit_runtime.cpp @@ -493,7 +493,9 @@ extern "C" { if ( !jit_module_is_registered(moduleName) ) { DAS_FATAL_ERROR("Failed to find %s: module %s is not registered (its .shared_module may have failed to load - see errors above).\n", funcMangledName, moduleName); } - DAS_FATAL_ERROR("Failed to find %s in module %s.\n", funcMangledName, moduleName); + das::string pending = describe_pending_dynamic_modules(); + DAS_FATAL_ERROR("Failed to find %s in module %s.%s%s\n", funcMangledName, moduleName, + pending.empty() ? "" : " Dynamic modules still pending: ", pending.c_str()); } } @@ -1189,12 +1191,73 @@ DAS_API void das_ensure_environment () { das::daScriptEnvironment::ensure(); } +DAS_API void * jit_register_module_once ( const char * dasName, das::Module * (*reg)() ) { + das::daScriptEnvironment::ensure(); + if ( das::Module * have = das::Module::require(dasName ? dasName : "") ) return have; + return reg(); +} + +DAS_API int32_t jit_lib_run_once ( int32_t * guard, void ** env, void (*fn)() ) { + static das::mutex once_mutex; + das::lock_guard lock(once_mutex); + if ( *guard ) { + if ( *env && das::daScriptEnvironment::getBound()!=*env ) { + das::daScriptEnvironment::setBound((das::daScriptEnvironment *)*env); + } + return 1; + } + das::daScriptEnvironment::ensure(); + *env = das::daScriptEnvironment::getBound(); + *guard = das::daScriptEnvironment::getBound()->modules ? 2 : 1; + fn(); + return 1; +} + +DAS_API int32_t jit_lib_invoke_guarded ( das::Context * ctx, void (*tramp)(das::Context *, void *), void * frame ) { + if ( !ctx ) return 0; + ctx->clearException(); + if ( ctx->contextMutex ) { + das::lock_guard guard(*ctx->contextMutex); + return ctx->runWithCatch([&]() { tramp(ctx, frame); }) ? 1 : 0; + } + return ctx->runWithCatch([&]() { tramp(ctx, frame); }) ? 1 : 0; +} + +DAS_API das::Context * jit_lib_create_finish ( das::Context * ctx, int32_t ok, char ** err ) { + if ( ok ) return ctx; + const char * why = ctx ? ( ctx->getException() ? ctx->getException() : "unknown exception" ) + : "out of memory"; + if ( err ) { + free(*err); + const size_t len = strlen(why) + 1; + *err = (char *) malloc(len); + if ( *err ) memcpy(*err, why, len); + } + delete ctx; + return nullptr; +} + +DAS_API const char * jit_lib_last_error ( das::Context * ctx ) { + return ctx ? ctx->getException() : nullptr; +} + +DAS_API void jit_destroy_standalone_ctx ( das::Context * ctx ) { + delete ctx; +} + DAS_API void jit_initialize_modules () { // No need to initialize modules. JIT will generate required calls. das::daScriptEnvironment::ensure(); } -DAS_API void jit_initialize_modules_done () { +DAS_API void jit_initialize_modules_done ( int32_t guard ) { + if ( guard==2 ) { + das::string notInitialized; + if ( !das::Module::InitializeDependencies(notInitialized) ) { + das::LOG(das::LogLevel::error) << "LLVM LIB: unable to initialize modules:" << notInitialized << "\n"; + } + return; + } das::Module::Initialize(); } @@ -1206,6 +1269,11 @@ DAS_API void jit_shutdown () { das::Module::ShutdownStandalone(); } +DAS_API void jit_lib_shutdown ( int32_t guard ) { + if ( guard!=1 ) return; + das::Module::ShutdownStandalone(); +} + DAS_API void * jit_register_dynamic_module ( const char * path, const char * mod_name ) { return das::register_dynamic_module(path, mod_name, 0/*Quiet*/, nullptr, nullptr); } @@ -1263,6 +1331,11 @@ DAS_API void jit_finalize_dynamic_modules () { } } +DAS_API void jit_lib_finalize_dynamic_modules () { + das::retry_pending_dynamic_modules(); + das::report_pending_dynamic_modules(); +} + // ABI shim: -exe binaries emitted before the resolving form link this runtime dynamically // and still import the 3-argument name. DAS_API void jit_register_native_path ( const char * mod_name, const char * src_path, const char * dst_path ) { diff --git a/src/builtin/module_builtin_rtti.cpp b/src/builtin/module_builtin_rtti.cpp index ddf1a04118..53c4f39355 100644 --- a/src/builtin/module_builtin_rtti.cpp +++ b/src/builtin/module_builtin_rtti.cpp @@ -974,6 +974,7 @@ namespace das { addField("no_lint"); addField("no_init_check"); addField("export_all"); + addField("export_public_functions"); addField("serialize_main_module"); addField("keep_alive"); addField("very_safe_context"); diff --git a/src/builtin/module_jit.cpp b/src/builtin/module_jit.cpp index 1d2af19fc8..aecbd6a49d 100644 --- a/src/builtin/module_jit.cpp +++ b/src/builtin/module_jit.cpp @@ -376,8 +376,31 @@ namespace das { #endif return run_link_cmd(cmd.c_str(), libraryName, "Library", context); } + + bool create_static_library ( const char * objFilePath, const char * libraryName, const char * customTool, Context * context ) { + if ( !check_file_present(objFilePath) ) { + LOG(LogLevel::error) << "File '" << objFilePath << "' , containing compiled definitions, does not exist\n"; + return false; + } + remove(libraryName); + std::string cmd; + #if defined(_WIN32) || defined(_WIN64) + #if defined(_MSC_VER) + const auto tool = find_linker(customTool, "llvm-lib.exe", "lib"); + cmd = fmt::format(FMT_STRING("\"\"{}\" /nologo /OUT:\"{}\" \"{}\" 2>&1\""), tool.c_str(), libraryName, objFilePath); + #else + const auto tool = find_linker(customTool, "llvm-ar.exe", "ar"); + cmd = fmt::format(FMT_STRING("\"\"{}\" rcs \"{}\" \"{}\" 2>&1\""), tool.c_str(), libraryName, objFilePath); + #endif + #else + const auto tool = find_linker(customTool, "llvm-ar", "ar"); + cmd = fmt::format(FMT_STRING("\"{}\" rcs \"{}\" \"{}\" 2>&1"), tool.c_str(), libraryName, objFilePath); + #endif + return run_link_cmd(cmd.c_str(), libraryName, "Archive", context); + } #else bool create_shared_library ( const char * objFilePath, const char * libraryName, [[maybe_unused]] const char * dasLib, const char * customLinker, const char * extraLinkerArgs, bool isShared, bool linkWholeLib, bool debugInfo, Context *context ) { return true; } + bool create_static_library ( const char * objFilePath, const char * libraryName, const char * customTool, Context * context ) { return true; } #endif // ===== --jit-split-modules parallel optimize+emit ===== @@ -748,6 +771,9 @@ namespace das { addExternInline(*this, lib, "create_shared_library", SideEffects::worstDefault, "create_shared_library") ->args({"objFilePath","libraryName","dasLib","customLinker","extraLinkerArgs","isShared","linkWholeLib","debugInfo","context"}); + addExternInline(*this, lib, "create_static_library", + SideEffects::worstDefault, "create_static_library") + ->args({"objFilePath","libraryName","customTool","context"}); addExternInline(*this, lib, "jit_par_emit_begin", SideEffects::worstDefault, "jit_par_emit_begin"); addExternInline(*this, lib, "jit_par_emit_add", diff --git a/tests-cpp/big/standalone_ctx/CMakeLists.txt b/tests-cpp/big/standalone_ctx/CMakeLists.txt index d36abaaeea..03ee09c248 100644 --- a/tests-cpp/big/standalone_ctx/CMakeLists.txt +++ b/tests-cpp/big/standalone_ctx/CMakeLists.txt @@ -67,8 +67,9 @@ add_dependencies(test-small test_standalone_capi) # The same emission, loaded instead of linked: a shared library built from the generated .cpp # exports the C entry points (default visibility; on Windows the header's _API macro is # what says so), and a daslang host reaches them through the bindings the same run wrote. Needs -# the shared runtime, since the host process already carries one. -if(TARGET libDaScriptDyn_runtime) +# the shared runtime, since the host process already carries one, and dasbind to load it - +# DAS_BIND_EXTERNAL is 0 on 32-bit Windows (das_config.h), where the host cannot bind at all. +if(TARGET libDaScriptDyn_runtime AND NOT (WIN32 AND CMAKE_SIZEOF_VOID_P EQUAL 4)) add_library(standalone_init_fixture_shared SHARED "${STANDALONE_CTX_GEN}/standalone_init_fixture.das.cpp") target_link_libraries(standalone_init_fixture_shared PRIVATE @@ -97,7 +98,7 @@ if(TARGET libDaScriptDyn_runtime) "${CMAKE_CURRENT_SOURCE_DIR}/test_standalone_bindings_host.das" WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}) set_tests_properties(standalone_capi_dasbind PROPERTIES LABELS "small") - add_dependencies(test-small standalone_init_fixture_shared) + add_dependencies(test-small standalone_init_fixture_shared daslang) endif() # The generated header asserts every structure's size and each of its field offsets. Nothing @@ -122,7 +123,8 @@ if(NOT MSVC) COMMENT "Standalone AOT: standalone_layout_fixture.das" VERBATIM ) - add_custom_target(standalone_layout_fixture_header + # ALL, not just a test-small dependency: CI runs `ctest -L small` without building that target + add_custom_target(standalone_layout_fixture_header ALL DEPENDS "${STANDALONE_CTX_GEN}/standalone_layout_fixture.das.h") set_target_properties(standalone_layout_fixture_header PROPERTIES FOLDER "tests-cpp/big") diff --git a/tests-cpp/small/test_jit_lib_guard.cpp b/tests-cpp/small/test_jit_lib_guard.cpp new file mode 100644 index 0000000000..383a9132bc --- /dev/null +++ b/tests-cpp/small/test_jit_lib_guard.cpp @@ -0,0 +1,114 @@ +#include + +#include "daScript/daScript.h" + +#include +#include +#include + +extern "C" { + DAS_API das::Context * jit_create_standalone_ctx ( uint64_t totalVariables, + uint64_t totalFunctions, + uint64_t globalStringHeapSize, + uint64_t globalsSize, + uint64_t sharedSize, + bool pinvoke, + uint64_t stackSize ); + DAS_API int32_t jit_lib_invoke_guarded ( das::Context * ctx, + void (*tramp)(das::Context *, void *), + void * frame ); + DAS_API das::Context * jit_lib_create_finish ( das::Context * ctx, int32_t ok, char ** err ); + DAS_API const char * jit_lib_last_error ( das::Context * ctx ); + DAS_API void jit_destroy_standalone_ctx ( das::Context * ctx ); + DAS_API int32_t jit_lib_run_once ( int32_t * guard, void ** env, void (*fn)() ); + DAS_API void * jit_register_module_once ( const char * dasName, das::Module * (*reg)() ); +} + +namespace { + +static int g_ran = 0; + +static void tramp_quiet ( das::Context *, void * frame ) { + if ( frame ) *(int *) frame = 7; + g_ran ++; +} + +static void tramp_raises ( das::Context * ctx, void * ) { + ctx->throw_error("boom in the body"); +} + +static int g_once_calls = 0; +static void bump_once () { g_once_calls ++; } + +static das::Context * make_ctx () { + return jit_create_standalone_ctx(0, 1, 0, 0, 0, false, 16 * 1024); +} + +} + +TEST_CASE("jit_lib_invoke_guarded runs a body and reports success") { + das::Context * ctx = make_ctx(); + REQUIRE(ctx != nullptr); + g_ran = 0; + int slot = 0; + CHECK(jit_lib_invoke_guarded(ctx, &tramp_quiet, &slot) == 1); + CHECK(g_ran == 1); + CHECK(slot == 7); + CHECK(jit_lib_last_error(ctx) == nullptr); + jit_destroy_standalone_ctx(ctx); +} + +TEST_CASE("jit_lib_invoke_guarded turns a das panic into a return code plus a message") { + das::Context * ctx = make_ctx(); + REQUIRE(ctx != nullptr); + CHECK(jit_lib_invoke_guarded(ctx, &tramp_raises, nullptr) == 0); + const char * err = jit_lib_last_error(ctx); + REQUIRE(err != nullptr); + CHECK(std::strstr(err, "boom in the body") != nullptr); + + int slot = 0; + CHECK(jit_lib_invoke_guarded(ctx, &tramp_quiet, &slot) == 1); + CHECK(slot == 7); + CHECK(jit_lib_last_error(ctx) == nullptr); + jit_destroy_standalone_ctx(ctx); +} + +TEST_CASE("jit_lib_create_finish drops a context whose init raised, and keeps the message") { + das::Context * ctx = make_ctx(); + REQUIRE(ctx != nullptr); + jit_lib_invoke_guarded(ctx, &tramp_raises, nullptr); + char * slot = nullptr; + CHECK(jit_lib_create_finish(ctx, 0, &slot) == nullptr); + REQUIRE(slot != nullptr); + CHECK(std::strstr(slot, "boom in the body") != nullptr); + + das::Context * good = make_ctx(); + REQUIRE(good != nullptr); + CHECK(jit_lib_create_finish(good, 1, &slot) == good); + CHECK(jit_lib_last_error(nullptr) == nullptr); + jit_destroy_standalone_ctx(good); + free(slot); +} + +TEST_CASE("jit_lib_run_once runs a library's init in a process that already carries a runtime") { + das::daScriptEnvironment::ensure(); + REQUIRE(das::daScriptEnvironment::getBound() != nullptr); + REQUIRE(das::daScriptEnvironment::getBound()->modules != nullptr); + g_once_calls = 0; + int32_t guard = 0; + void * env = nullptr; + CHECK(jit_lib_run_once(&guard, &env, &bump_once) == 1); + CHECK(g_once_calls == 1); + CHECK(guard == 2); + CHECK(env == (void *) das::daScriptEnvironment::getBound()); + CHECK(jit_lib_run_once(&guard, &env, &bump_once) == 1); + CHECK(g_once_calls == 1); + CHECK(das::daScriptEnvironment::getBound()->modules != nullptr); +} + +TEST_CASE("jit_register_module_once hands back a module the process already registered") { + das::daScriptEnvironment::ensure(); + das::Module * have = das::Module::require("math"); + REQUIRE(have != nullptr); + CHECK(jit_register_module_once("math", nullptr) == (void *) have); +} diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 15ed8e2bdf..5f31ee22e2 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -115,3 +115,6 @@ if(TARGET test_llvm_aot) USES_TERMINAL ) endif() + +# Two standalone sweeps over the test corpus, one per backend. Opt-in, out of CI. +add_subdirectory(standalone-sweep) diff --git a/tests/jit_tests/_jit_lib_align.das b/tests/jit_tests/_jit_lib_align.das new file mode 100644 index 0000000000..4f1e2a226b --- /dev/null +++ b/tests/jit_tests/_jit_lib_align.das @@ -0,0 +1,15 @@ +options gen2 + + +require daslib/export_c + +struct Wide { + a : int8 + b : int64 + c : float +} + +[export_c] +def make(n : int) : Wide { + return Wide(a = int8(n), b = int64(n) * 1000000000l, c = float(n) + 0.5) +} diff --git a/tests/jit_tests/_jit_lib_align_host.das b/tests/jit_tests/_jit_lib_align_host.das new file mode 100644 index 0000000000..5ac4747fb6 --- /dev/null +++ b/tests/jit_tests/_jit_lib_align_host.das @@ -0,0 +1,22 @@ +options gen2 + +require daslib/safe_addr +require _jit_lib_align_c + +[export] +def main() : int { + var ctx = jit_lib_align_create() + if (ctx == null) { + print("create failed: {jit_lib_align_last_error(null)}\n") + return 1 + } + var bad = 0 + var w = jit_lib_align_Wide(a = int8(0), b = 0l, c = 0.0) + jit_lib_align_make(ctx, 7, safe_addr(w)) + if (w.a != int8(7) || w.b != 7000000000l || w.c != 7.5) { + print("make answered a={w.a} b={w.b} c={w.c}\n") + bad++ + } + jit_lib_align_destroy(ctx) + return bad +} diff --git a/tests/jit_tests/_jit_lib_bind.das b/tests/jit_tests/_jit_lib_bind.das new file mode 100644 index 0000000000..78e224f803 --- /dev/null +++ b/tests/jit_tests/_jit_lib_bind.das @@ -0,0 +1,35 @@ +options gen2 + + +require daslib/export_c + +struct Pair { + lo : int + hi : int +} + +var g_calls = 0 + +[export_c] +def bump() : int { + g_calls++ + return g_calls +} + +[export_c] +def widen(p : Pair) : Pair { + return Pair(lo = p.lo - 1, hi = p.hi + 1) +} + +[export_c] +def greet(who : string) : string { + return "hola {who}" +} + +[export_c(name = "double_or_raise")] +def boom(n : int) : int { + if (n < 0) { + panic("negative: {n}") + } + return n * 2 +} diff --git a/tests/jit_tests/_jit_lib_bind_host.das b/tests/jit_tests/_jit_lib_bind_host.das new file mode 100644 index 0000000000..ab7208f220 --- /dev/null +++ b/tests/jit_tests/_jit_lib_bind_host.das @@ -0,0 +1,47 @@ +options gen2 + +require daslib/safe_addr +require strings +require _bindlib_c + +[export] +def main() : int { + var ctx = bindlib_create() + if (ctx == null) { + print("create failed: {bindlib_last_error(null)}\n") + return 1 + } + var bad = 0 + if (bindlib_bump(ctx) != 1 || bindlib_bump(ctx) != 2) { + print("the library's global does not persist across calls\n") + bad++ + } + var p = bindlib_Pair(lo = 10, hi = 20) + var out = bindlib_Pair(lo = 0, hi = 0) + bindlib_widen(ctx, safe_addr(p), safe_addr(out)) + if (out.lo != 9 || out.hi != 21) { + print("widen answered ({out.lo},{out.hi})\n") + bad++ + } + let greeting = bindlib_greet(ctx, "mundo") + if (greeting != "hola mundo") { + print("greet answered `{greeting}`\n") + bad++ + } + if (bindlib_double_or_raise(ctx, 21) != 42) { + print("a plain call answered wrong\n") + bad++ + } + let raised = bindlib_double_or_raise(ctx, -5) + let err = bindlib_last_error(ctx) + if (raised != 0 || find(err, "negative: -5") < 0) { + print("a raising call answered {raised}, last_error `{err}`\n") + bad++ + } + if (bindlib_double_or_raise(ctx, 3) != 6) { + print("the instance is unusable after a panic\n") + bad++ + } + bindlib_destroy(ctx) + return bad +} diff --git a/tests/jit_tests/_jit_lib_export_all_host.das b/tests/jit_tests/_jit_lib_export_all_host.das new file mode 100644 index 0000000000..294930c56b --- /dev/null +++ b/tests/jit_tests/_jit_lib_export_all_host.das @@ -0,0 +1,20 @@ +options gen2 + +require _jit_lib_probe_all_c + +[export] +def main() : int { + var ctx = jit_lib_probe_all_create() + if (ctx == null) { + print("create failed: {jit_lib_probe_all_last_error(null)}\n") + return 1 + } + var bad = 0 + let bumped = jit_lib_probe_all_helper_public(ctx, 41) + if (bumped != 42) { + print("helper_public answered {bumped}\n") + bad++ + } + jit_lib_probe_all_destroy(ctx) + return bad +} diff --git a/tests/jit_tests/_jit_lib_guest.das b/tests/jit_tests/_jit_lib_guest.das new file mode 100644 index 0000000000..e83eac5a36 --- /dev/null +++ b/tests/jit_tests/_jit_lib_guest.das @@ -0,0 +1,16 @@ +options gen2 + + +require daslib/ast +require daslib/export_c +require UnitTest + +[export_c] +def probe(n : int) : int { + return n + 7 +} + +[export_c] +def touch_ast : bool { + return compiling_module() == null +} diff --git a/tests/jit_tests/_jit_lib_guest_host.das b/tests/jit_tests/_jit_lib_guest_host.das new file mode 100644 index 0000000000..908635c4b1 --- /dev/null +++ b/tests/jit_tests/_jit_lib_guest_host.das @@ -0,0 +1,20 @@ +options gen2 + +require _jit_lib_guest_c + +[export] +def main() : int { + var ctx = jit_lib_guest_create() + if (ctx == null) { + print("create failed: {jit_lib_guest_last_error(null)}\n") + return 1 + } + var bad = 0 + let answered = jit_lib_guest_probe(ctx, 35) + if (answered != 42) { + print("probe answered {answered}, last_error `{jit_lib_guest_last_error(ctx)}`\n") + bad++ + } + jit_lib_guest_destroy(ctx) + return bad +} diff --git a/tests/jit_tests/_jit_lib_probe.das b/tests/jit_tests/_jit_lib_probe.das new file mode 100644 index 0000000000..affb4e3d09 --- /dev/null +++ b/tests/jit_tests/_jit_lib_probe.das @@ -0,0 +1,32 @@ +options gen2 + + +require daslib/export_c + +struct Pair { + lo : int + hi : int +} + +var g_seen = 0 + +[export_c] +def widen(p : Pair) : Pair { + g_seen++ + return Pair(lo = p.lo - 1, hi = p.hi + 1) +} + +[export_c(name = "tally")] +def count_seen() : int { + return g_seen +} + +[export_c] +def mapped(n : int) : int { + let fx <- @@(x : int) => x * 3 + return invoke(fx, n) +} + +def helper_public(x : int) : int { + return x + 1 +} diff --git a/tests/jit_tests/_jit_lib_probe_host.das b/tests/jit_tests/_jit_lib_probe_host.das new file mode 100644 index 0000000000..3ad52195fd --- /dev/null +++ b/tests/jit_tests/_jit_lib_probe_host.das @@ -0,0 +1,33 @@ +options gen2 + +require daslib/safe_addr +require _jit_lib_probe_c + +[export] +def main() : int { + var ctx = jit_lib_probe_create() + if (ctx == null) { + print("create failed: {jit_lib_probe_last_error(null)}\n") + return 1 + } + var bad = 0 + var p = jit_lib_probe_Pair(lo = 10, hi = 20) + var out = jit_lib_probe_Pair(lo = 0, hi = 0) + jit_lib_probe_widen(ctx, safe_addr(p), safe_addr(out)) + if (out.lo != 9 || out.hi != 21) { + print("widen answered ({out.lo},{out.hi})\n") + bad++ + } + let seen = jit_lib_probe_tally(ctx) + if (seen != 1) { + print("tally answered {seen}\n") + bad++ + } + let mapped = jit_lib_probe_mapped(ctx, 5) + if (mapped != 15) { + print("mapped answered {mapped}\n") + bad++ + } + jit_lib_probe_destroy(ctx) + return bad +} diff --git a/tests/jit_tests/_jit_lib_shared_host.das b/tests/jit_tests/_jit_lib_shared_host.das new file mode 100644 index 0000000000..3b68c3c29f --- /dev/null +++ b/tests/jit_tests/_jit_lib_shared_host.das @@ -0,0 +1,35 @@ +options gen2 + +require daslib/safe_addr +require _standalone_init_fixture_c + +[export] +def main() : int { + var ctx = standalone_init_fixture_create() + if (ctx == null) { + print("create failed: {standalone_init_fixture_last_error(null)}\n") + return 1 + } + var bad = 0 + let first = standalone_init_fixture_get_first(ctx) + let second = standalone_init_fixture_get_second(ctx) + let stamp = standalone_init_fixture_get_init_fn_stamp(ctx) + let shared_total = standalone_init_fixture_get_shared_total(ctx) + if (first != 31 || second != 2 || stamp != 3 || shared_total != 6) { + print("globals answered first={first} second={second} stamp={stamp} shared={shared_total}\n") + bad++ + } + var p = standalone_init_fixture_Pair(a = 40, b = 2) + let inner = standalone_init_fixture_Inner(weight = 9) + var o = standalone_init_fixture_Outer(inner = inner, tag = 5) + let pair_sum = standalone_init_fixture_pair_sum(ctx, safe_addr(p)) + let outer = standalone_init_fixture_outer_weight(ctx, safe_addr(o)) + let head = standalone_init_fixture_head_value(ctx) + let renamed = standalone_init_fixture_renamed_sum(ctx, 20, 22) + if (pair_sum != 42 || outer != 14 || head != 41 || renamed != 42) { + print("calls answered pair_sum={pair_sum} outer={outer} head={head} renamed={renamed}\n") + bad++ + } + standalone_init_fixture_destroy(ctx) + return bad +} diff --git a/tests/jit_tests/_jit_lib_two_module_sets_host.das b/tests/jit_tests/_jit_lib_two_module_sets_host.das new file mode 100644 index 0000000000..91116ec1f8 --- /dev/null +++ b/tests/jit_tests/_jit_lib_two_module_sets_host.das @@ -0,0 +1,32 @@ +options gen2 + +require _standalone_modules_fixture_c +require _jit_lib_guest_c + +[export] +def main() : int { + var mods = standalone_modules_fixture_create() + if (mods == null) { + print("the fio library refused to create: {standalone_modules_fixture_last_error(null)}\n") + return 1 + } + var guest = jit_lib_guest_create() + if (guest == null) { + print("the ast library refused to create: {jit_lib_guest_last_error(null)}\n") + standalone_modules_fixture_destroy(mods) + return 1 + } + var bad = 0 + if (!standalone_modules_fixture_has_path_variable(mods)) { + print("the fio library cannot read the environment\n") + bad++ + } + let answered = jit_lib_guest_probe(guest, 35) + if (answered != 42) { + print("the ast library answered {answered}\n") + bad++ + } + jit_lib_guest_destroy(guest) + standalone_modules_fixture_destroy(mods) + return bad +} diff --git a/tests/jit_tests/jit_lib.das b/tests/jit_tests/jit_lib.das new file mode 100644 index 0000000000..f12e87c1b8 --- /dev/null +++ b/tests/jit_tests/jit_lib.das @@ -0,0 +1,218 @@ +options gen2 +options no_aot + +require dastest/testing_boost +require daslib/fio +require daslib/strings_boost +require strings + + +let OUTPUT_DIR = "{get_das_root()}/build/tests" +let FIXTURE_DIR = "{get_das_root()}/tests/jit_tests" +let SHARED_FIXTURE = "{get_das_root()}/tests-cpp/big/standalone_ctx/standalone_init_fixture.das" + + +def private spawn(cmd : string; var lines : array) : int { + var rc : int + unsafe { + rc = popen_timeout("{cmd} 2>&1", 600.0) $(f) { + if (f == null) { + return + } + while (!feof(f)) { + let ln = strip(fgets(f)) + if (!(ln |> empty())) { + lines |> push("{ln}") + } + } + } + } + return rc +} + + +def private shared_artifact(stem : string) : string { + let plat = get_platform_name() + if (plat == "windows") { + return "{stem}.dll" + } + return plat == "darwin" ? "{stem}.dylib" : "{stem}.so" +} + + +def private static_artifact(stem : string) : string { + return get_platform_name() == "windows" ? "{stem}.lib" : "{stem}.a" +} + + +def private build_lib(bin, source, stem, flags, jit_flags : string; var lines : array) : int { + mkdir_rec(OUTPUT_DIR) + return spawn("\"{bin}\" -lib \"{source}\" -output \"{OUTPUT_DIR}/{stem}\" {flags}" + + " -- {jit_flags} --jit-lib-bindings \"{FIXTURE_DIR}/_{stem}_c.das\"", lines) +} + + +def private run_host(bin, host : string; var lines : array) : int { + return spawn("cd \"{get_das_root()}\" && \"{bin}\" \"{FIXTURE_DIR}/{host}\"", lines) +} + + +def private report(lines : array; what : string) { + for (ln in lines) { + to_log(LOG_ERROR, "{what}: {ln}\n") + } +} + + +def private lib_tier_ready(t : T?) : bool { + if (!jit_enabled() || !das_is_dll_build()) { + to_log(LOG_WARNING, "jit_lib: SKIPPED - -lib needs the LLVM backend and a shared runtime\n") + t |> success(true, "-lib needs the LLVM backend and a shared runtime to link against") + return false + } + return true +} + + +[test] +def test_jit_lib_shared(t : T?) { + if (!lib_tier_ready(t)) { + return + } + let args <- get_command_line_arguments() + let bin = args[0] + var lines : array + let rc = build_lib(bin, "{FIXTURE_DIR}/_jit_lib_probe.das", "jit_lib_probe", "", "", lines) + report(lines, "build") + t |> success(rc == 0, "daslang -lib exits clean") + t |> success(fexist("{OUTPUT_DIR}/jit_lib_probe.h"), "the C header is written beside the library") + t |> success(fexist(shared_artifact("{OUTPUT_DIR}/jit_lib_probe")), "the shared library is written") + var host_lines : array + let host_rc = run_host(bin, "_jit_lib_probe_host.das", host_lines) + report(host_lines, "host") + t |> success(host_rc == 0, "a struct, a renamed call and a lambda-using body all answer through C") +} + + +[test] +def test_jit_lib_static_and_export_all(t : T?) { + if (!lib_tier_ready(t)) { + return + } + let args <- get_command_line_arguments() + let bin = args[0] + var static_lines : array + let static_rc = build_lib(bin, "{FIXTURE_DIR}/_jit_lib_probe.das", "jit_lib_probe_s", + "-lib-export-all", "--jit-lib-static", static_lines) // nolint:LINT029 - lines carries the build transcript for the failure report + report(static_lines, "static build") + t |> success(static_rc == 0, "a static export-all build exits clean") + t |> success(fexist(static_artifact("{OUTPUT_DIR}/jit_lib_probe_s")), "the archive is written") + + var lines : array + let rc = build_lib(bin, "{FIXTURE_DIR}/_jit_lib_probe.das", "jit_lib_probe_all", + "-lib-export-all", "", lines) + report(lines, "build") + t |> success(rc == 0, "a shared export-all build exits clean") + var host_lines : array + let host_rc = run_host(bin, "_jit_lib_export_all_host.das", host_lines) + report(host_lines, "host") + t |> success(host_rc == 0, "export-all reaches a bare public function, under the output name's prefix") +} + + +[test] +def test_jit_lib_result_slot_is_aligned(t : T?) { + if (!lib_tier_ready(t)) { + return + } + let args <- get_command_line_arguments() + let bin = args[0] + var lines : array + let rc = build_lib(bin, "{FIXTURE_DIR}/_jit_lib_align.das", "jit_lib_align", "", "", lines) + report(lines, "build") + t |> success(rc == 0, "the alignment fixture builds") + var host_lines : array + let host_rc = run_host(bin, "_jit_lib_align_host.das", host_lines) + report(host_lines, "host") + t |> success(host_rc == 0, "a structure whose alignment exceeds the argument frame's crosses intact") +} + + +[test] +def test_jit_lib_binds_into_a_daslang_host(t : T?) { + if (!lib_tier_ready(t)) { + return + } + let args <- get_command_line_arguments() + let bin = args[0] + var lines : array + let rc = build_lib(bin, "{FIXTURE_DIR}/_jit_lib_bind.das", "bindlib", "", "", lines) + report(lines, "build") + t |> success(rc == 0, "the bind library builds") + t |> success(fexist("{FIXTURE_DIR}/_bindlib_c.das"), "the daslang bindings are written") + var host_lines : array + let host_rc = run_host(bin, "_jit_lib_bind_host.das", host_lines) + report(host_lines, "host") + t |> success(host_rc == 0, "a global, a struct, a string and a panic all behave through the bindings") +} + + +[test] +def test_jit_lib_runs_the_standalone_fixture(t : T?) { + if (!lib_tier_ready(t)) { + return + } + let args <- get_command_line_arguments() + let bin = args[0] + var lines : array + let rc = build_lib(bin, SHARED_FIXTURE, "standalone_init_fixture", "", "--jit-lib-export-marked", lines) + report(lines, "build") + t |> success(rc == 0, "the AOT standalone fixture also builds as a JIT library") + var host_lines : array + let host_rc = run_host(bin, "_jit_lib_shared_host.das", host_lines) + report(host_lines, "host") + t |> success(host_rc == 0, "both tiers answer the same values from the same source") +} + + +[test] +def test_jit_lib_is_a_guest_of_a_populated_runtime(t : T?) { + if (!lib_tier_ready(t)) { + return + } + let args <- get_command_line_arguments() + let bin = args[0] + var lines : array + let rc = build_lib(bin, "{FIXTURE_DIR}/_jit_lib_guest.das", "jit_lib_guest", "", "", lines) + report(lines, "build") + t |> success(rc == 0, "the guest library builds") + var host_lines : array + let host_rc = run_host(bin, "_jit_lib_guest_host.das", host_lines) + report(host_lines, "host") + t |> success(host_rc == 0, "a library reaching its own modules survives a populated runtime") +} + + +[test] +def test_jit_lib_carries_two_module_sets(t : T?) { + if (!lib_tier_ready(t)) { + return + } + let args <- get_command_line_arguments() + let bin = args[0] + var fio_lines : array + let fio_rc = build_lib(bin, "{get_das_root()}/tests-cpp/big/standalone_ctx/standalone_modules_fixture.das", + "standalone_modules_fixture", "", "--jit-lib-export-marked", fio_lines) + report(fio_lines, "fio library") + t |> success(fio_rc == 0, "the AOT modules fixture also builds as a JIT library") + var guest_lines : array + let guest_rc = build_lib(bin, "{FIXTURE_DIR}/_jit_lib_guest.das", "jit_lib_guest", "", "", guest_lines) + report(guest_lines, "ast library") + t |> success(guest_rc == 0, "the ast library builds") + var host_lines : array + let host_rc = run_host(bin, "_jit_lib_two_module_sets_host.das", host_lines) + report(host_lines, "host") + t |> success(host_rc == 0, "two libraries with different module sets answer in one process") +} + + diff --git a/tests/standalone-sweep/CMakeLists.txt b/tests/standalone-sweep/CMakeLists.txt new file mode 100644 index 0000000000..90c11e6b5a --- /dev/null +++ b/tests/standalone-sweep/CMakeLists.txt @@ -0,0 +1,96 @@ +# Two standalone sweeps over test_aot's corpus, one per backend. Opt-in, out of CI: +# ninja standalone_sweep_aot / run_standalone_sweep_aot -ctx: compile, link, launch all +# ninja standalone_sweep_jit -lib: emit, load back, drive each +# -DDAS_STANDALONE_SWEEP_FILTER= narrows the corpus. + +if(NOT TARGET daslang OR NOT DEFINED TEST_AOT_ALL_DAS) + return() +endif() + +set(DAS_STANDALONE_SWEEP_FILTER "" CACHE STRING + "Regex narrowing the standalone sweep corpus (empty = every file test_aot covers)") + +# DAS_AOT_CTX resolves against the tree root, not this subdirectory +set(_sweep_in "") +foreach(_f IN LISTS TEST_AOT_ALL_DAS) + if(IS_ABSOLUTE ${_f}) + list(APPEND _sweep_in ${_f}) + else() + list(APPEND _sweep_in ${PROJECT_SOURCE_DIR}/${_f}) + endif() +endforeach() +list(REMOVE_DUPLICATES _sweep_in) +list(FILTER _sweep_in EXCLUDE REGEX "/_") # required helpers, not entry points +if(NOT DAS_STANDALONE_SWEEP_FILTER STREQUAL "") + list(FILTER _sweep_in INCLUDE REGEX "${DAS_STANDALONE_SWEEP_FILTER}") +endif() + +# `options no_aot`, a quote and require_module_now each leave a file with no standalone form; +# a stem another file already claimed would emit one namespace twice and collide at link +set(_sweep_files "") +set(_sweep_stems "") +set(_skipped 0) +foreach(_f IN LISTS _sweep_in) + get_filename_component(_stem ${_f} NAME_WE) + file(STRINGS ${_f} _no_form REGEX "^[ \t]*options[ \t].*no_aot|\\[no_aot\\]|qmacro|qmatch|qblock|quote\\(|require_module_now") + if(_no_form OR (_stem IN_LIST _sweep_stems)) + math(EXPR _skipped "${_skipped} + 1") + file(RELATIVE_PATH _rel ${PROJECT_SOURCE_DIR} ${_f}) + message(VERBOSE "standalone sweep: not swept - ${_rel}") + else() + list(APPEND _sweep_stems ${_stem}) + list(APPEND _sweep_files ${_f}) + endif() +endforeach() +list(LENGTH _sweep_files _sweep_count) +message(STATUS "standalone sweep: ${_sweep_count} files, ${_skipped} with no standalone form or a claimed stem") +list(JOIN _sweep_files "|" _sweep_files_arg) + +### C++ tier +add_custom_target(standalone_sweep_aot_corpus) +set(SWEEP_CTX_GENERATED_SRC) +DAS_AOT_CTX("${_sweep_files}" SWEEP_CTX_GENERATED_SRC standalone_sweep_aot_corpus daslang) + +# which files emitted an entry point is known only after the generate step +set(_contexts "${CMAKE_CURRENT_BINARY_DIR}/sweep_contexts.h") +add_custom_command( + OUTPUT ${_contexts} + COMMAND ${CMAKE_COMMAND} -DOUT=${_contexts} "-DFILES=${_sweep_files_arg}" + -P ${CMAKE_CURRENT_SOURCE_DIR}/sweep_contexts.cmake + DEPENDS standalone_sweep_aot_corpus ${CMAKE_CURRENT_SOURCE_DIR}/sweep_contexts.cmake + COMMENT "Standalone sweep (C++): collecting the contexts that emitted" + VERBATIM +) + +add_executable(standalone_sweep_aot EXCLUDE_FROM_ALL + "${CMAKE_CURRENT_SOURCE_DIR}/sweep_driver.c" ${_contexts} ${SWEEP_CTX_GENERATED_SRC}) +target_link_libraries(standalone_sweep_aot PRIVATE libDaScript ${SRC_LIBRARIES} ${DAS_MODULES_LIBS}) +# the same corpus test_aot compiles, so the same include list +target_include_directories(standalone_sweep_aot PRIVATE + "${CMAKE_CURRENT_BINARY_DIR}" ${TEST_AOT_INCLUDE_DIRS} ${DAS_CONFIG_INCLUDE_DIR}) +set_target_properties(standalone_sweep_aot PROPERTIES + C_STANDARD 11 C_STANDARD_REQUIRED ON LINKER_LANGUAGE CXX FOLDER tests) +SETUP_CPP11(standalone_sweep_aot) +add_dependencies(standalone_sweep_aot standalone_sweep_aot_corpus) + +add_custom_target(run_standalone_sweep_aot + COMMAND $ + DEPENDS standalone_sweep_aot + WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} + COMMENT "Standalone sweep (C++): launching every context" + USES_TERMINAL +) + +### JIT tier +if(NOT DAS_LLVM_DISABLED) + add_custom_target(standalone_sweep_jit + COMMAND ${CMAKE_COMMAND} -DDASLANG=$ -DROOT=${PROJECT_SOURCE_DIR} + -DOUT=${CMAKE_CURRENT_BINARY_DIR}/_lib "-DFILES=${_sweep_files_arg}" + -P ${CMAKE_CURRENT_SOURCE_DIR}/sweep_jit.cmake + DEPENDS daslang + WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} + COMMENT "Standalone sweep (JIT -lib): every file as a C-ABI library, loaded back" + USES_TERMINAL + VERBATIM + ) +endif() diff --git a/tests/standalone-sweep/sweep_contexts.cmake b/tests/standalone-sweep/sweep_contexts.cmake new file mode 100644 index 0000000000..78a7b6b16e --- /dev/null +++ b/tests/standalone-sweep/sweep_contexts.cmake @@ -0,0 +1,49 @@ +# sweep_contexts.h for sweep_driver.c: one include per context that emitted an entry point, the +# rest as an X-macro. Contexts linking C++ modules run first and one is held open for the sweep - +# they bind the thread environment in a base class, and the global AOT library pass needs it. + +string(REPLACE "|" ";" _files "${FILES}") +set(_includes "") +set(_list "") +set(_with_modules "") +set(_skipped 0) +foreach(_f IN LISTS _files) + get_filename_component(_dir ${_f} DIRECTORY) + get_filename_component(_name ${_f} NAME) + set(_gen "${_dir}/_standalone_ctx_generated/${_name}") + set(_entry "") + if(EXISTS "${_gen}.h") + # the prefix escapes a keyword stem (enum -> enum_), so read it rather than assume it + file(STRINGS "${_gen}.h" _entry REGEX "_ctx \\* [A-Za-z_][A-Za-z0-9_]*_create\\(void\\)" LIMIT_COUNT 1) + endif() + string(REGEX MATCH "_ctx \\* ([A-Za-z_][A-Za-z0-9_]*)_create" _unused "${_entry}") + if(NOT _entry OR CMAKE_MATCH_1 STREQUAL "") + math(EXPR _skipped "${_skipped} + 1") + continue() + endif() + string(APPEND _includes "#include \"${_gen}.h\"\n") + set(_mods "") + if(EXISTS "${_gen}.cpp") + file(STRINGS "${_gen}.cpp" _mods REGEX "standalone_modules\\.h" LIMIT_COUNT 1) + endif() + if(_mods) + string(APPEND _with_modules " RUN(${CMAKE_MATCH_1}); \\\n") + else() + string(APPEND _list " RUN(${CMAKE_MATCH_1}); \\\n") + endif() +endforeach() + +set(_anchor_call "") +if(NOT _with_modules STREQUAL "") + string(REGEX MATCH "RUN\\(([A-Za-z_][A-Za-z0-9_]*)\\)" _unused "${_with_modules}") + set(_anchor_call "A(${CMAKE_MATCH_1})") +endif() + +file(WRITE ${OUT} "/* Generated by sweep_contexts.cmake. DO NOT EDIT. */ +${_includes} +#define SWEEP_ANCHOR(A) ${_anchor_call} +#define SWEEP_CONTEXTS(RUN) \\ +${_with_modules}${_list} /* end */ + +") +message(STATUS "standalone sweep: ${_skipped} contexts skipped by the emitter") diff --git a/tests/standalone-sweep/sweep_driver.c b/tests/standalone-sweep/sweep_driver.c new file mode 100644 index 0000000000..4124a14371 --- /dev/null +++ b/tests/standalone-sweep/sweep_driver.c @@ -0,0 +1,30 @@ + +#include + +#include "sweep_contexts.h" + +static int sweep_total = 0; +static int sweep_failed = 0; + +#define RUN(P) \ + do { \ + ++sweep_total; \ + P##_ctx * ctx = P##_create(); \ + if (ctx) { \ + P##_destroy(ctx); \ + } else { \ + ++sweep_failed; \ + printf("FAIL %s: %s\n", #P, P##_last_error(0)); \ + } \ + } while (0) + +#define HOLD(P) P##_ctx * anchor = P##_create() +#define DROP(P) if (anchor) P##_destroy(anchor) + +int main ( void ) { + SWEEP_ANCHOR(HOLD); + SWEEP_CONTEXTS(RUN) + SWEEP_ANCHOR(DROP); + printf("standalone sweep (C++): %d contexts, %d failed\n", sweep_total, sweep_failed); + return sweep_failed ? 1 : 0; +} diff --git a/tests/standalone-sweep/sweep_jit.cmake b/tests/standalone-sweep/sweep_jit.cmake new file mode 100644 index 0000000000..17e949c70f --- /dev/null +++ b/tests/standalone-sweep/sweep_jit.cmake @@ -0,0 +1,112 @@ +# The JIT half of the standalone sweep, run by the standalone_sweep_jit target: +# cmake -DDASLANG= -DROOT= -DOUT=

"-DFILES=a.das|b.das" -P sweep_jit.cmake +# Emission goes through utils/internal/jit/main.das, which takes many files per process - the +# ~52-file dasLLVM load costs ~4.5s and is paid once per chunk, not once per library. Each library +# is then loaded back by a generated host. A library that never emitted is tallied as refused; one +# that emitted and cannot run sets the exit code. + +foreach(_var DASLANG ROOT OUT FILES) + if(NOT DEFINED ${_var}) + message(FATAL_ERROR "sweep_jit.cmake: -D${_var} is required") + endif() +endforeach() +file(MAKE_DIRECTORY ${OUT}) +string(REPLACE "|" ";" _files "${FILES}") +list(LENGTH _files _total) + +# the emitters are .das, so daslang's mtime says nothing about them +file(GLOB _emitter_srcs ${ROOT}/daslib/*.das ${ROOT}/modules/dasLLVM/daslib/*.das + ${ROOT}/utils/internal/jit/main.das) +set(_newest_input "${DASLANG}") +foreach(_src IN LISTS _emitter_srcs) + if(${_src} IS_NEWER_THAN ${_newest_input}) + set(_newest_input "${_src}") + endif() +endforeach() + +set(_emit "") +set(_kept 0) +foreach(_f IN LISTS _files) + get_filename_component(_stem ${_f} NAME_WE) + set(_so "${OUT}/${_stem}.so") + if(EXISTS ${_so} AND EXISTS "${OUT}/${_stem}_c.das" + AND NOT ${_f} IS_NEWER_THAN ${_so} AND NOT ${_newest_input} IS_NEWER_THAN ${_so}) + math(EXPR _kept "${_kept} + 1") + else() + list(APPEND _emit ${_f}) + endif() +endforeach() +list(LENGTH _emit _n_emit) +message(STATUS "standalone sweep (JIT): ${_n_emit} libraries to emit, ${_kept} reused") + +# 32 per process: a file that leaves the JIT's global state broken takes only its chunk with it +set(_chunk "") +set(_done 0) +foreach(_f IN LISTS _emit) + list(APPEND _chunk ${_f}) + math(EXPR _done "${_done} + 1") + list(LENGTH _chunk _n_chunk) + if(_n_chunk EQUAL 32 OR _done EQUAL _n_emit) + execute_process( + COMMAND ${DASLANG} ${ROOT}/utils/internal/jit/main.das -- ${_chunk} + --jit-lib --lib-export-all --lib-output-dir ${OUT} --jit-lib-bindings auto + WORKING_DIRECTORY ${ROOT}) + set(_chunk "") + endif() +endforeach() + +set(_ok 0) +set(_refused "") +set(_broken "") +foreach(_f IN LISTS _files) + get_filename_component(_stem ${_f} NAME_WE) + set(_bind "${OUT}/${_stem}_c.das") + set(_p "") + if(EXISTS ${_bind}) + # the entry points carry the emitter's prefix, which escapes a keyword stem (enum -> enum_) + file(STRINGS ${_bind} _decl REGEX "name=\"[A-Za-z_][A-Za-z0-9_]*_create\"" LIMIT_COUNT 1) + string(REGEX MATCH "name=\"([A-Za-z_][A-Za-z0-9_]*)_create\"" _unused "${_decl}") + set(_p "${CMAKE_MATCH_1}") + endif() + if(_p STREQUAL "") + list(APPEND _refused ${_f}) + continue() + endif() + + # the host binds nothing by hand: it requires the generated bindings beside it, and + # shutdown_runtime balances what create brought up, or daslang's atexit check trips + set(_host "${OUT}/${_stem}_host.das") + file(WRITE ${_host} +"options gen2 + +require ${_stem}_c + +[export] +def main() { + var ctx = ${_p}_create() + if (ctx == null) { + panic(\"${_p}: {${_p}_last_error(null)}\") + } + ${_p}_destroy(ctx) + ${_p}_shutdown_runtime() +} +") + execute_process(COMMAND ${DASLANG} ${_host} WORKING_DIRECTORY ${ROOT} + RESULT_VARIABLE _rc OUTPUT_VARIABLE _log ERROR_VARIABLE _log) + if(_rc EQUAL 0) + math(EXPR _ok "${_ok} + 1") + else() + list(APPEND _broken ${_f}) + message(STATUS " FAILED to load or run (${_rc})\n${_log}") + endif() +endforeach() + +list(LENGTH _refused _n_refused) +list(LENGTH _broken _n_broken) +message(STATUS "standalone sweep (JIT): ${_total} files, ${_ok} ran, ${_n_refused} refused, ${_n_broken} broken") +foreach(_f IN LISTS _refused _broken) + message(STATUS " not run: ${_f}") +endforeach() +if(_n_broken GREATER 0) + message(FATAL_ERROR "standalone sweep (JIT): ${_n_broken} libraries built but did not run") +endif() diff --git a/utils/daslang/main.cpp b/utils/daslang/main.cpp index 2252250d97..55102f1cd2 100644 --- a/utils/daslang/main.cpp +++ b/utils/daslang/main.cpp @@ -62,11 +62,14 @@ enum class JitMode { Direct, Dll, Executable, + Library, }; static JitMode jitEnabled = JitMode::None; // Disabled by default. static bool jitNoCache = false; // -jit-no-cache: bypass DLL-cache path, run in-memory. static bool jitStack = false; // -jit-stack: retain every generated call in the logical das stack. static string jitOutPath = ""; // Empty, JIT module will choose default. +static bool libExportAll = false; +static bool libNeedsOutput = false; static string serFile = ""; // -ser : write the AST module cache (env serializer rail) after compile static string deserFile = ""; // -deser : read the AST module cache during compile instead of parsing static string moduleCacheFile = ""; // -module-cache : both - read when present, refresh when the compile diverged @@ -473,6 +476,9 @@ int compile_and_run ( const string & fn, const string & mainFnName, bool outputP policies.jit_enabled = true; switch (jitEnabled) { case JitMode::Executable: policies.jit_exe_mode = true; break; + case JitMode::Library: + policies.export_public_functions = libExportAll; + break; case JitMode::Dll: policies.jit_dll_mode = true; break; case JitMode::Direct: break; default: break; @@ -590,6 +596,9 @@ int compile_and_run ( const string & fn, const string & mainFnName, bool outputP return 0; } + if ( jitEnabled==JitMode::Library ) { + program->options.push_back(AnnotationArgument("jit_lib", true)); + } auto simulate0 = ref_time_ticks(); auto pctx = SimulateWithErrReport(program, tout); startupSimulateUsec += get_time_usec(simulate0); @@ -730,6 +739,10 @@ void print_help() { << " Useful when the cached .jitted_scripts/ DLL is stale or unwanted.\n" << " -jit-stack with -jit: retain every generated call in the logical daslang stack.\n" << " -exe JIT compile to standalone executable (implies -dry-run)\n" + << " -lib JIT compile to a C-ABI native library: .so/.dylib/.dll plus .h\n" + << " (add -- --jit-lib-static for a .a/.lib archive instead; implies -dry-run)\n" + << " -lib-export-all with -lib: export every public entry-module function whose signature has a C\n" + << " representation, instead of only the [export_c] ones\n" << " -output set JIT output path\n" << " --list-shared-modules with -exe: write JSON describing the program's shared modules and daspkg-package .das module sources to \n" << " --force-shared-module with -exe: force-include a shared module by daslang or package name (repeatable)\n" @@ -933,6 +946,12 @@ int MAIN_FUNC_NAME ( int argc, char * argv[] ) { } else if ( cmd=="exe") { jitEnabled = JitMode::Executable; dryRun = true; + } else if ( cmd=="lib") { + jitEnabled = JitMode::Library; + dryRun = true; + libNeedsOutput = true; + } else if ( cmd=="lib-export-all") { + libExportAll = true; } else if ( cmd=="ser" ) { if ( i+1 >= argc ) { printf("-ser requires path argument\n"); @@ -1117,6 +1136,11 @@ int MAIN_FUNC_NAME ( int argc, char * argv[] ) { printf("-no-module-cache disables the cache; do not combine it with -ser/-deser\n"); return -1; } + if ( libNeedsOutput && jitOutPath.empty() ) { + printf("-lib needs -output : a host includes the generated header by name, and the\n" + "default JIT cache path is hash-named and swept\n"); + return -1; + } startupPreScanUsec = get_time_usec(startupMain0); auto builtin0 = ref_time_ticks(); // register modules diff --git a/utils/internal/jit/main.das b/utils/internal/jit/main.das index d676337ef1..fd94ee03da 100644 --- a/utils/internal/jit/main.das +++ b/utils/internal/jit/main.das @@ -28,6 +28,14 @@ struct JitToolArgs { @clarg_doc = "Output path. Empty keeps the default content-hashed .jitted_scripts// naming (dll mode) or appends .exe (exe mode)." output : string + @clarg_name = "lib-output-dir" + @clarg_doc = "Batch --jit-lib: each input's library lands as /, so one process emits many libraries - which is what amortizes the ~52-file dasLLVM compile-time load. --jit-lib itself is read by the backend, from the same command line" + lib_output_dir : string + + @clarg_name = "lib-export-all" + @clarg_doc = "Batch --jit-lib: export every public function whose signature C can spell, instead of only the [export_c] ones" + lib_export_all : bool + @clarg_name = "aot-object-prefix" @clarg_doc = "Batch --aot-object: derive each input's object as /_llvm_aot_generated/_.o, so one process emits many objects (amortizes the ~52-file dasLLVM compile-time load). Per-input; overrides --output." aot_object_prefix : string @@ -263,6 +271,41 @@ def run_parallel(files : array; tool : JitToolArgs; jit : JitCliOptions) return rc } +def private jit_setup_cop(var cop : CodeOfPolicies; input : string; tool : JitToolArgs; jit : JitCliOptions) : string { + cop.jit_enabled = true + cop.version_2_syntax = true + cop.aot_module = true + cop.threadlock_context = true + if (jit.lib |> unwrap_or(false)) { + cop.jit_dll_mode = false + cop.export_public_functions = tool.lib_export_all + } elif (tool.aot_object) { + cop.jit_emit_object = true + cop.jit_dll_mode = false + } elif (tool.exe) { + cop.jit_exe_mode = true + cop.jit_dll_mode = false + } else { + cop.jit_dll_mode = true + } + var out_path = ((tool.aot_object && !empty(tool.aot_object_prefix)) + ? "{dir_name(input)}/_llvm_aot_generated/{tool.aot_object_prefix}_{base_name(input)}" + : tool.output) + if (!empty(tool.lib_output_dir)) { + let stem = (base_name(input) |> split("."))[0] + out_path = "{tool.lib_output_dir}/{stem}" + } + if (!empty(out_path)) { + cop.jit_output_path := out_path + } + cop.jit_opt_level = jit.opt_level |> unwrap_or(cop.jit_opt_level) + cop.jit_size_level = jit.size_level |> unwrap_or(cop.jit_size_level) + cop.jit_debug_info = jit.debug_info |> unwrap_or(cop.jit_debug_info) + cop.emit_prologue = jit.stack_frames |> unwrap_or(cop.emit_prologue) + return out_path +} + + def jit_compile_one(input : string; tool : JitToolArgs; jit : JitCliOptions) : bool { var success = true ast_gc_guard() { @@ -270,34 +313,7 @@ def jit_compile_one(input : string; tool : JitToolArgs; jit : JitCliOptions) : b access |> add_extra_module("just_in_time", "{get_das_root()}/daslib/just_in_time.das") using() $(var mg : ModuleGroup) { using() $(var cop : CodeOfPolicies) { - cop.jit_enabled = true - cop.version_2_syntax = true - // Match dastest's JIT setup: aot_module prepares finalizers / - // lambdas / generators for AOT-JIT codegen (without it, the JIT - // can't emit IR for them -> "Failed to get IR for functions ..."). - cop.aot_module = true - cop.threadlock_context = true - if (tool.aot_object) { - cop.jit_emit_object = true - cop.jit_dll_mode = false - } elif (tool.exe) { - cop.jit_exe_mode = true - cop.jit_dll_mode = false - } else { - cop.jit_dll_mode = true - } - // Batch object mode derives a per-input path so one process emits many objects; - // else use the single --output (dll/exe or one-shot object). - let out_path = ((tool.aot_object && !empty(tool.aot_object_prefix)) - ? "{dir_name(input)}/_llvm_aot_generated/{tool.aot_object_prefix}_{base_name(input)}" - : tool.output) - if (!empty(out_path)) { - cop.jit_output_path := out_path - } - cop.jit_opt_level = jit.opt_level |> unwrap_or(cop.jit_opt_level) - cop.jit_size_level = jit.size_level |> unwrap_or(cop.jit_size_level) - cop.jit_debug_info = jit.debug_info |> unwrap_or(cop.jit_debug_info) - cop.emit_prologue = jit.stack_frames |> unwrap_or(cop.emit_prologue) + let out_path = jit_setup_cop(cop, input, tool, jit) let ptl = jit.path_to_linker |> unwrap_or("") if (!empty(ptl)) { cop.jit_path_to_linker := ptl @@ -327,7 +343,9 @@ def jit_compile_one(input : string; tool : JitToolArgs; jit : JitCliOptions) : b if (success) { let target = tool.aot_object ? "AOT object" : (tool.exe ? "executable" : "shared library") if (!empty(out_path)) { - let suffix = tool.aot_object ? ".o" : (tool.exe ? ".exe" : ".dll") + let plat = get_platform_name() + let shared_ext = plat == "windows" ? ".dll" : (plat == "darwin" ? ".dylib" : ".so") + let suffix = tool.aot_object ? ".o" : (tool.exe ? ".exe" : shared_ext) to_log(LOG_INFO, "jit: wrote {target} for {input} -> {out_path}{suffix}\n") } else { to_log(LOG_INFO, "jit: wrote {target} for {input} under .jitted_scripts/{ns}/\n") From 5e0ef9a43cef27f6c56e4d67eb9b463c3864fdf0 Mon Sep 17 00:00:00 2001 From: Churkin Aleksey Date: Sat, 12 Sep 2026 00:41:42 +0300 Subject: [PATCH 3/3] TEMP: win64-only run that bisects standalone_capi_dasbind - REVERT The matrix is pinned to the one cell that fails, every other job in build.yml is off, and every other workflow drops its pull_request trigger, so one run answers one question. The debug step runs the host directly rather than through ctest, so its output survives, and walks the call sequence: bindings loaded and nothing called, then create alone, then create plus destroy, then a scalar call between them, then the real host. Whichever step first exits 0xC0000374 names the call that corrupts the heap. dumpbin /dependents on daslang.exe and on the fixture DLL answers the other open question - whether the two sides link the same CRT. Revert with the run it explains. --- .github/workflows/build.yml | 118 +++++++++++++++--- .github/workflows/build_eastl.yml | 5 +- .github/workflows/codeql.yml | 18 +-- .github/workflows/cpp_mcp_release.yml | 10 +- .github/workflows/dasllama-server-e2e.yml | 13 +- .github/workflows/dasllama_server_release.yml | 13 +- .github/workflows/doc.yml | 19 +-- .github/workflows/extended_checks.yml | 5 +- .github/workflows/fatman.yml | 9 +- .github/workflows/pages.yml | 3 +- .github/workflows/playground-e2e.yml | 13 +- .github/workflows/vulkan_checks.yml | 4 +- .github/workflows/wasm_build.yml | 5 +- .github/workflows/wasmboy.yml | 9 +- 14 files changed, 123 insertions(+), 121 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f1e3d8e4f2..4718d5e0bd 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -58,7 +58,8 @@ jobs: # two statements: a substitution inside echo would hide the script's exit code run: | set -eu - matrix=$(python3 ci/ci_matrix.py build '${{ github.event_name }}') + # TEMP: one cell only - win64 Release, the lane standalone_capi_dasbind fails on + matrix='{"include": [{"target": "windows", "architecture": 64, "cmake_preset": "Release", "sanitizers": "none", "release_target": "windows", "release_arch": "x86_64", "runner": "windows-latest", "architecture_string": "x64", "archive_ext": "zip", "llvm_disabled": "ON", "jit_disabled": "ON", "build_system": "cmake", "cmake_generator": "Ninja"}]}' echo "matrix=$matrix" >> "$GITHUB_OUTPUT" - name: Cache LLVM @@ -294,7 +295,9 @@ jobs: # daslang's MSVC debug info is /Z7 (CMakeCommon.txt) so sccache # can cache it. cmake --no-warn-unused-cli -B./build -G "${{ matrix.cmake_generator }}" -DCMAKE_BUILD_TYPE:STRING=${{ matrix.cmake_preset }} -DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache -DDAS_LLVM_DISABLED=${{ env.das_llvm_disabled }} -DCMAKE_TOOLCHAIN_FILE="$VCPKG_ROOT/scripts/buildsystems/vcpkg.cmake" - cmake --build ./build --config ${{ matrix.cmake_preset }} --parallel + # TEMP: the debug step needs daslang and the fixture DLL, nothing else. + # Building ALL here costs about an hour of AOT translation units. + cmake --build ./build --config ${{ matrix.cmake_preset }} --parallel --target daslang standalone_init_fixture_shared ;; linux_arm*) CC=clang CXX=clang++ cmake --no-warn-unused-cli -B./build -DCMAKE_BUILD_TYPE:STRING=${{ matrix.cmake_preset }} -DDAS_GLFW_DISABLED=ON -DDAS_HV_DISABLED=OFF -DDAS_SQLITE_DISABLED=OFF -DDAS_LLVM_DISABLED=${{ env.das_llvm_disabled }} -G \ @@ -332,6 +335,7 @@ jobs: # warm sequential sweep, so a separate prewarm only duplicated the # frontend-compile of every test. The target still exists for local use. - name: "Test" + if: false # TEMP: debug run only needs the DEBUG step run: | set -eux # When daslang is sanitizer-instrumented, its JIT-emitted .dll cache @@ -410,7 +414,94 @@ jobs: cmake --build ./build --config ${{ matrix.cmake_preset }} --target run_tests_interpreter || cmake --build ./build --config ${{ matrix.cmake_preset }} --target run_tests_interpreter_isolated ;; esac + - name: "TEMP DEBUG: standalone_capi_dasbind on win64" + if: always() + shell: bash + run: | + set -x + H=tests-cpp/big/standalone_ctx + D=build/tests-cpp/big/standalone_ctx + BIN=bin/daslang.exe + + echo "===== which CRT does each side link =====" + dumpbin //dependents "$BIN" || true + dumpbin //dependents "$D/standalone_init_fixture.dll" || true + + echo "===== 0: bindings required, nothing called =====" + cat > "$H/_dbg0.das" <<'EOS' + options gen2 + require _standalone_init_fixture_c + [export] + def main : int { + print("loaded, no call\n") + return 0 + } + EOS + "$BIN" "$H/_dbg0.das" || true; echo "dbg0 exit=$?" + + echo "===== 1: create only, no destroy =====" + cat > "$H/_dbg1.das" <<'EOS' + options gen2 + require _standalone_init_fixture_c + [export] + def main : int { + print("before create\n") + var ctx = standalone_init_fixture_create() + print("created null={ctx == null}\n") + return 0 + } + EOS + "$BIN" "$H/_dbg1.das" || true; echo "dbg1 exit=$?" + + echo "===== 2: create + destroy =====" + cat > "$H/_dbg2.das" <<'EOS' + options gen2 + require _standalone_init_fixture_c + [export] + def main : int { + var ctx = standalone_init_fixture_create() + print("created null={ctx == null}\n") + standalone_init_fixture_destroy(ctx) + print("destroyed\n") + return 0 + } + EOS + "$BIN" "$H/_dbg2.das" || true; echo "dbg2 exit=$?" + + echo "===== 3: create + scalar call + destroy =====" + cat > "$H/_dbg3.das" <<'EOS' + options gen2 + require _standalone_init_fixture_c + [export] + def main : int { + var ctx = standalone_init_fixture_create() + let first = standalone_init_fixture_get_first(ctx) + print("get_first={first}\n") + standalone_init_fixture_destroy(ctx) + print("destroyed\n") + return 0 + } + EOS + "$BIN" "$H/_dbg3.das" || true; echo "dbg3 exit=$?" + + echo "===== 4: the real host, with its output =====" + "$BIN" "$H/test_standalone_bindings_host.das" || true; echo "host exit=$?" + + echo "===== 4b: a stack for the create crash, under cdb =====" + CDB="/c/Program Files (x86)/Windows Kits/10/Debuggers/x64/cdb.exe" + ls "$CDB" || CDB=$(find "/c/Program Files (x86)/Windows Kits" -name cdb.exe 2>/dev/null | head -1) + if [ -n "$CDB" ] && [ -x "$CDB" ]; then + "$CDB" -g -G -c 'g; .lastevent; k 40; !heap -p -a @rcx; q' \ + "$(cygpath -w "$PWD/bin/daslang.exe")" "$(cygpath -w "$PWD/$H/_dbg1.das")" 2>&1 | tail -80 || true + else + echo "no cdb on this runner" + fi + + echo "===== 5: through ctest, for the record =====" + ctest --test-dir build --build-config Release -R standalone_capi_dasbind --output-on-failure || true + - name: "Small C++ Tests" + if: false # TEMP: debug run only needs the DEBUG step run: | set -eux # Suppress LSan false-positives from the Uri binding; see "Test" step above @@ -437,7 +528,7 @@ jobs: # per-PR ctest is -L small, so only this step executes a generated # context. memory_model_4gb allocates a real 4 GB chunk and stays # local-only. - if: matrix.cmake_preset == 'Release' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') + if: false # TEMP: debug run only needs the DEBUG step run: | set -eux case "${{ matrix.target }}${{ matrix.architecture }}" in @@ -466,9 +557,7 @@ jobs: # duplicates the Release/Debug compile and interpreter coverage on every # PR. Keep it on the canonical nightly and explicit full-workflow runs. needs: pre_job - if: >- - (github.event_name == 'schedule' && github.repository == 'GaijinEntertainment/daScript') - || github.event_name == 'workflow_dispatch' + if: false # TEMP: win64 debug run runs-on: windows-latest permissions: contents: read @@ -560,6 +649,7 @@ jobs: run: cmake --build ./build --config RelWithDebInfo --target run_tests_interpreter - name: "Small C++ Tests" + if: false # TEMP: debug run only needs the DEBUG step run: ctest --test-dir build --build-config RelWithDebInfo -L small --output-on-failure ########################################################### @@ -570,9 +660,7 @@ jobs: # The regular matrix cell is the fast LLVM-free MSVC gate. Its scheduled # run still owns the full AOT suite, so do not duplicate that sweep here. needs: pre_job - if: >- - (github.event_name == 'schedule' && github.repository == 'GaijinEntertainment/daScript') - || github.event_name == 'workflow_dispatch' + if: false # TEMP: win64 debug run runs-on: windows-latest permissions: contents: read @@ -652,6 +740,7 @@ jobs: cmake --build ./build --config Release --target run_tests_interpreter || cmake --build ./build --config Release --target run_tests_interpreter_isolated - name: "Small C++ Tests" + if: false # TEMP: debug run only needs the DEBUG step run: ctest --test-dir build --build-config Release -L small --output-on-failure ########################################################### @@ -668,9 +757,7 @@ jobs: # Per-PR/push lane and manual dispatch, plus the nightly cron on the canonical repo: the cron # is what seeds this job's sccache slot below (a master push run is usually pre_job-skipped # as a concurrent duplicate of the PR that just merged, so its save rarely lands). - if: >- - (github.event_name == 'schedule' && github.repository == 'GaijinEntertainment/daScript') - || (github.event_name != 'schedule' && needs.pre_job.outputs.should_skip != 'true') + if: false # TEMP: win64 debug run runs-on: ubuntu-latest # actions: write needed for `gh cache delete` in the sccache refresh step. permissions: @@ -778,7 +865,7 @@ jobs: # scheduled cron runs the toolchains only on the canonical repo, so forks # don't run (and fail) the nightly — that would email every fork owner. # Manual workflow_dispatch still runs them anywhere. - if: (github.event_name == 'schedule' && github.repository == 'GaijinEntertainment/daScript') || github.event_name == 'workflow_dispatch' + if: false # TEMP: win64 debug run runs-on: windows-latest defaults: run: @@ -867,6 +954,7 @@ jobs: || cmake --build ./build-mingw --config Release --target run_tests_jit_isolated - name: "Small C++ Tests" + if: false # TEMP: debug run only needs the DEBUG step run: | set -eux cd build-mingw @@ -935,7 +1023,7 @@ jobs: # scheduled cron runs the toolchains only on the canonical repo, so forks # don't run (and fail) the nightly — that would email every fork owner. # Manual workflow_dispatch still runs them anywhere. - if: (github.event_name == 'schedule' && github.repository == 'GaijinEntertainment/daScript') || github.event_name == 'workflow_dispatch' + if: false # TEMP: win64 debug run runs-on: windows-latest steps: - name: "SCM Checkout" @@ -1037,7 +1125,7 @@ jobs: # (core + HV/SQLITE/Audio/PUGIXML/UnitTest modules) is covered without it. ########################################################### needs: pre_job - if: needs.pre_job.outputs.should_skip != 'true' && github.event_name != 'schedule' + if: false # TEMP: win64 debug run runs-on: ubuntu-latest steps: - name: "SCM Checkout" diff --git a/.github/workflows/build_eastl.yml b/.github/workflows/build_eastl.yml index 1e05e61814..936a0b920a 100644 --- a/.github/workflows/build_eastl.yml +++ b/.github/workflows/build_eastl.yml @@ -1,9 +1,8 @@ name: build_eastl on: - push: - branches: [master] - pull_request: + # TEMP: push trigger off for the win64 debug run + # TEMP: pull_request trigger off for the win64 debug run workflow_dispatch: # Compile-coverage for the dagor config: EASTL containers, dag::Vector as diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index f0df555278..a3d9655fb6 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -8,22 +8,8 @@ name: "CodeQL" # CodeQL — that surface is covered by the in-tree lint. on: - push: - branches: [master] - paths: - - 'src/**' - - 'include/**' - - 'modules/**' - - 'tests-cpp/**' - - '.github/workflows/codeql.yml' - pull_request: - branches: [master] - paths: - - 'src/**' - - 'include/**' - - 'modules/**' - - 'tests-cpp/**' - - '.github/workflows/codeql.yml' + # TEMP: push trigger off for the win64 debug run + # TEMP: pull_request trigger off for the win64 debug run schedule: # weekly full refresh keeps the master baseline current even when no # C++-touching push happens (PR alert diffing compares against it) diff --git a/.github/workflows/cpp_mcp_release.yml b/.github/workflows/cpp_mcp_release.yml index a52ebd45fc..83b46b6bd9 100644 --- a/.github/workflows/cpp_mcp_release.yml +++ b/.github/workflows/cpp_mcp_release.yml @@ -15,15 +15,7 @@ name: cpp-mcp release # modules, so every external module (dasHV/OpenSSL, GLFW, Audio, ...) is disabled # — no vcpkg/openssl/apt module deps, and a ~360-step build instead of 2000+. on: - pull_request: - paths: - - 'utils/mcp/**' - - 'utils/common/**' - - 'tree-sitter-daslang/*.yml' # ast-grep rule files the bundle ships - - 'ci/make_cpp_mcp_bundle.sh' - - 'ci/smoke_test_cpp_mcp.sh' - - '.github/workflows/cpp_mcp_release.yml' - - 'CMakeLists.txt' + # TEMP: pull_request trigger off for the win64 debug run workflow_dispatch: release: types: [prereleased] diff --git a/.github/workflows/dasllama-server-e2e.yml b/.github/workflows/dasllama-server-e2e.yml index 188fcea64a..8b38f0aa66 100644 --- a/.github/workflows/dasllama-server-e2e.yml +++ b/.github/workflows/dasllama-server-e2e.yml @@ -4,17 +4,8 @@ name: dasllama-server control page e2e # fixtures; regeneration rail: utils/dasllama-server/tests/fixtures/README.md. on: - pull_request: - paths: - - 'utils/dasllama-server/control.html' - - 'utils/dasllama-server/tests/**' - - '.github/workflows/dasllama-server-e2e.yml' - push: - branches-ignore: [master] - paths: - - 'utils/dasllama-server/control.html' - - 'utils/dasllama-server/tests/**' - - '.github/workflows/dasllama-server-e2e.yml' + # TEMP: pull_request trigger off for the win64 debug run + # TEMP: push trigger off for the win64 debug run workflow_dispatch: jobs: diff --git a/.github/workflows/dasllama_server_release.yml b/.github/workflows/dasllama_server_release.yml index dde839a49c..672162143c 100644 --- a/.github/workflows/dasllama_server_release.yml +++ b/.github/workflows/dasllama_server_release.yml @@ -23,18 +23,7 @@ name: dasllama-server release # it, and 22.04's (glibc 2.35) admits Debian 12 and every later Ubuntu, where 24.04's (2.39) # refused Debian 12 with `GLIBC_2.38 not found`. on: - push: - branches-ignore: [master] - paths: - - '.github/workflows/dasllama_server_release.yml' - - 'utils/dasllama-server/**' - - 'utils/watchdog/**' - - 'utils/daspkg/**' - - 'daslib/daspkg.das' - - 'modules/dasHV/**' - - 'modules/dasLLAMA/benchmarks/lcpp_bench.das' - - 'modules/dasLLAMA/dasllama/dasllama_bench.das' - - '!**/*.md' + # TEMP: push trigger off for the win64 debug run workflow_dispatch: inputs: publish: diff --git a/.github/workflows/doc.yml b/.github/workflows/doc.yml index 7738fb4b6b..c7f326f597 100644 --- a/.github/workflows/doc.yml +++ b/.github/workflows/doc.yml @@ -4,23 +4,8 @@ name: Build doc on: release: types: prereleased - push: - paths: - - 'doc/**' - - 'daslib/**' - - 'src/builtin/**' - - 'modules/dasLLAMA/dasllama/**' - - 'modules/dasImgui/**' - - 'modules/dasVulkan/**' - pull_request: - paths: - - 'doc/**' - - 'daslib/**' - - 'src/builtin/**' - - 'modules/dasLLAMA/dasllama/**' - - 'modules/dasImgui/**' - - 'modules/dasVulkan/**' - + # TEMP: push trigger off for the win64 debug run + # TEMP: pull_request trigger off for the win64 debug run concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true diff --git a/.github/workflows/extended_checks.yml b/.github/workflows/extended_checks.yml index 0c4b729fce..91078f94bd 100644 --- a/.github/workflows/extended_checks.yml +++ b/.github/workflows/extended_checks.yml @@ -1,9 +1,8 @@ name: extended checks on: - push: - branches: [master] - pull_request: + # TEMP: push trigger off for the win64 debug run + # TEMP: pull_request trigger off for the win64 debug run # 04:00 UTC — after build.yml (02:00), the daspkg index (02:30) and imgui (03:00). # The nightly is what SEEDS the sccache slots below. Their save is master-only, # but the master PUSH run is usually skipped by pre_job as a concurrent duplicate diff --git a/.github/workflows/fatman.yml b/.github/workflows/fatman.yml index dcfd6a8ab5..2274b22021 100644 --- a/.github/workflows/fatman.yml +++ b/.github/workflows/fatman.yml @@ -7,14 +7,7 @@ name: fatman on: - pull_request: - paths: - - 'CMakeLists.txt' - - 'web/CMakeLists.txt' - - 'examples/fatman/**' - - '.github/workflows/fatman.yml' - - 'include/**' - - 'src/**' + # TEMP: pull_request trigger off for the win64 debug run workflow_dispatch: concurrency: diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml index 0528be01e4..5894bc8db0 100644 --- a/.github/workflows/pages.yml +++ b/.github/workflows/pages.yml @@ -1,8 +1,7 @@ name: Deploy daslang.io on: - push: - branches: [master] + # TEMP: push trigger off for the win64 debug run workflow_dispatch: # Only one site deployment at a time diff --git a/.github/workflows/playground-e2e.yml b/.github/workflows/playground-e2e.yml index 6ccdc7bacc..2979d078d9 100644 --- a/.github/workflows/playground-e2e.yml +++ b/.github/workflows/playground-e2e.yml @@ -6,17 +6,8 @@ name: Playground e2e (no WASM) # job wasm_specs). on: - pull_request: - paths: - - 'site/**' - - 'web/examples/ui/**' - - '.github/workflows/playground-e2e.yml' - push: - branches-ignore: [master] - paths: - - 'site/**' - - 'web/examples/ui/**' - - '.github/workflows/playground-e2e.yml' + # TEMP: pull_request trigger off for the win64 debug run + # TEMP: push trigger off for the win64 debug run workflow_dispatch: jobs: diff --git a/.github/workflows/vulkan_checks.yml b/.github/workflows/vulkan_checks.yml index f5c2c1d07c..5d998981e5 100644 --- a/.github/workflows/vulkan_checks.yml +++ b/.github/workflows/vulkan_checks.yml @@ -6,9 +6,7 @@ name: dasVulkan checks # and the module-wide lint. The full render suite is nightly_vulkan.yml. on: - pull_request: - paths: - - 'modules/dasVulkan/**' + # TEMP: pull_request trigger off for the win64 debug run workflow_dispatch: concurrency: diff --git a/.github/workflows/wasm_build.yml b/.github/workflows/wasm_build.yml index db44fe83c0..a3cab28cb1 100644 --- a/.github/workflows/wasm_build.yml +++ b/.github/workflows/wasm_build.yml @@ -1,9 +1,8 @@ name: wasm_build on: - push: - branches: [master] - pull_request: + # TEMP: push trigger off for the win64 debug run + # TEMP: pull_request trigger off for the win64 debug run workflow_dispatch: concurrency: diff --git a/.github/workflows/wasmboy.yml b/.github/workflows/wasmboy.yml index c396619aa2..d968060b4a 100644 --- a/.github/workflows/wasmboy.yml +++ b/.github/workflows/wasmboy.yml @@ -10,14 +10,7 @@ name: wasmboy on: - pull_request: - paths: - - 'CMakeLists.txt' - - 'web/**' - - 'examples/fatman/**' - - '.github/workflows/wasmboy.yml' - - 'include/**' - - 'src/**' + # TEMP: pull_request trigger off for the win64 debug run workflow_dispatch: concurrency: