diff --git a/backends/native/README.md b/backends/native/README.md index 9ce1b69d277..8c51b12efac 100644 --- a/backends/native/README.md +++ b/backends/native/README.md @@ -206,7 +206,12 @@ and API. `native_cpu_engine_test` covers the CPU engine when `EXECUTORCH_BUILD_NATIVE_CPU` is enabled. `native_vulkan_passes_test` covers the Vulkan graph rewrites without requiring a GPU. When either runner below is enabled, `native_runner_utils_test` covers their shared runner utilities. Run these -with `ctest --test-dir cmake-out -R native_`. +with `ctest --test-dir cmake-out -R native_` after `cmake --build cmake-out`. + +The engines' sample tests judge them on the bundled op samples. Building them runs +the sample generator, which needs `torch` and `torchao`. Those with failing cases +are disabled in CTest; run their binaries directly with +`--packages=/backends/native/test/op_samples`. The `Test Native Engines` CI workflow has separate Linux jobs for the shared runtime, CPU, GGML, and Vulkan, with each test suite run once. To reproduce with diff --git a/backends/native/runtime/deserialize/Samples.h b/backends/native/runtime/deserialize/Samples.h index bbbac2035c6..e2159616d09 100644 --- a/backends/native/runtime/deserialize/Samples.h +++ b/backends/native/runtime/deserialize/Samples.h @@ -23,6 +23,8 @@ struct SampleTensor { ScalarType dtype = kFloat; std::vector sizes; std::vector bytes; + + bool operator==(const SampleTensor&) const = default; }; struct Tolerance { diff --git a/backends/native/test/CMakeLists.txt b/backends/native/test/CMakeLists.txt index f6a72ffde51..719edc54578 100644 --- a/backends/native/test/CMakeLists.txt +++ b/backends/native/test/CMakeLists.txt @@ -65,6 +65,31 @@ add_custom_command( ) add_custom_target(native_op_samples DEPENDS ${_native_op_samples}.stamp) +if(EXECUTORCH_BUILD_NATIVE_GGML OR EXECUTORCH_BUILD_NATIVE_VULKAN) + if(NOT TARGET gflags AND NOT TARGET gflags::gflags) + find_package(gflags REQUIRED) + endif() + if(TARGET gflags::gflags) + set(_native_test_gflags gflags::gflags) + else() + set(_native_test_gflags gflags) + endif() + + # A gtest main judging the engine whose host a test links on the bundled + # samples of the packages it names; engine tests set its flags. + add_library(native_sample_test_main OBJECT samples/sample_test_main.cpp) + target_link_libraries( + native_sample_test_main + PUBLIC native_output_check native_runtime native_package native_engine + ${_native_test_gflags} GTest::gtest + ) + target_compile_options( + native_sample_test_main + PRIVATE $<$:/EHsc> + $<$>:-fexceptions> + ) +endif() + if(EXECUTORCH_BUILD_NATIVE_VULKAN) add_executable( native_vulkan_passes_test @@ -86,6 +111,23 @@ if(EXECUTORCH_BUILD_NATIVE_VULKAN) PRIVATE $<$:/EHsc> $<$>:-fexceptions> ) + + add_executable(native_vulkan_sample_test runtime/vulkan/vulkan_ptn_runner.cpp) + target_link_libraries( + native_vulkan_sample_test PRIVATE native_sample_test_main + native_vulkan_engine + ) + add_dependencies(native_vulkan_sample_test native_op_samples) + add_test(NAME native_vulkan_sample_test + COMMAND native_vulkan_sample_test --packages=${_native_op_samples} + ) + # TODO: Enable once every case passes. + set_tests_properties(native_vulkan_sample_test PROPERTIES DISABLED TRUE) + target_compile_options( + native_vulkan_sample_test + PRIVATE $<$:/EHsc> + $<$>:-fexceptions> + ) endif() if(EXECUTORCH_BUILD_NATIVE_GGML) @@ -101,6 +143,23 @@ if(EXECUTORCH_BUILD_NATIVE_GGML) PRIVATE $<$:/EHsc> $<$>:-fexceptions> ) + + add_executable(native_ggml_sample_test runtime/ggml/ggml_ptn_runner.cpp) + target_link_libraries( + native_ggml_sample_test PRIVATE native_sample_test_main native_ggml_engine + ) + add_dependencies(native_ggml_sample_test native_op_samples) + add_test(NAME native_ggml_sample_test + COMMAND native_ggml_sample_test --packages=${_native_op_samples} + --device=cpu + ) + # TODO: Enable once every case passes. + set_tests_properties(native_ggml_sample_test PROPERTIES DISABLED TRUE) + target_compile_options( + native_ggml_sample_test + PRIVATE $<$:/EHsc> + $<$>:-fexceptions> + ) endif() if(TARGET native_runner_utils) diff --git a/backends/native/test/runtime/ggml/BUCK b/backends/native/test/runtime/ggml/BUCK index 05de9fa2b72..71d998be09e 100644 --- a/backends/native/test/runtime/ggml/BUCK +++ b/backends/native/test/runtime/ggml/BUCK @@ -1,5 +1,6 @@ load("@fbcode_macros//build_defs:build_file_migration.bzl", "fbcode_target") load("@fbsource//third-party/llama.cpp:ggml_ci.bzl", "ggml_ci_labels") +load("@fbsource//tools/target_determinator/macros:ci.bzl", "ci") load("@fbsource//xplat/executorch/backends/native/runtime/engine:targets.bzl", "CPU_COMPAT", "CUDA_COMPAT", "HIP_COMPAT") load("@fbcode_macros//build_defs/lib:re_test_utils.bzl", "re_test_utils") load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "runtime") @@ -23,7 +24,7 @@ fbcode_target( ) # The op suite runs once per ggml backend set; only the engine variant, the -# runner's --device, link deps and resources, the test host, and the CI +# runner's --device, link deps and test env, the test host, and the CI # configuration differ. _VARIANTS = [ {"suffix": "", "engine": "ggml_engine_cpu", "compat": CPU_COMPAT, "device": "cpu"}, @@ -40,10 +41,9 @@ _VARIANTS = [ # beside it, and its VMM pool calls the CUDA driver API. "deps": ["fbsource//third-party/llama.cpp:ggml_base[prefer-shared]"], "external_deps": [("cuda", None, "cuda_driver_api-lazy")], - # Buck does not put the CUDA driver stub's library on the runner's - # loader path, so it ships beside the runner. - "resources": { - "//tools/lazify:lazy_stub_binder[shared]": "libtools_lazify_lazy_stub_binder.so", + "env": { + # Buck does not put the CUDA driver stub's library on the loader path. + # @oss-disable[end= ]: "LD_PRELOAD": "$(location //tools/lazify:lazy_stub_binder[shared])", }, }, { @@ -91,16 +91,34 @@ _VARIANTS = [ for v in _VARIANTS ] +[ + fbcode_target( + _kind = runtime.cxx_test, + name = "ggml_sample_test" + v["suffix"], + args = [ + "--packages=$(location //executorch/backends/native/test/samples:op_samples)", + "--device=" + v["device"], + ], + env = v.get("env", {}), + remote_execution = v.get("remote_execution"), + # TODO: run in CI once they pass. + labels = ci.labels(ci.map(ci.skip_test())), + test_main = "//executorch/backends/native/test/samples:sample_test_main", + deps = [":ggml_engine_host" + v["suffix"]], + **v.get("compat", {}) + ) + for v in _VARIANTS +] + [ fbcode_target( _kind = runtime.python_test, name = "test_ggml_ops" + v["suffix"], srcs = ["test_ggml_ops.py"], - env = {"GGML_PTN_DEVICE": v["device"]}, + env = {"GGML_PTN_DEVICE": v["device"]} | v.get("env", {}), labels = v.get("labels", []), remote_execution = v.get("remote_execution"), - resources = {":ggml_ptn_runner" + v["suffix"]: "ggml_ptn_runner"} | - v.get("resources", {}), + resources = {":ggml_ptn_runner" + v["suffix"]: "ggml_ptn_runner"}, deps = [ "//caffe2:torch", "//executorch/exir/native:native", diff --git a/backends/native/test/runtime/ggml/test_ggml_ops.py b/backends/native/test/runtime/ggml/test_ggml_ops.py index 54d31653a77..bdabbf10a72 100644 --- a/backends/native/test/runtime/ggml/test_ggml_ops.py +++ b/backends/native/test/runtime/ggml/test_ggml_ops.py @@ -334,18 +334,12 @@ def _run_ggml( command.append(f"--input_sizes={';'.join(map(_shape_text, steps))}") if graph_ops is not None: command.append("--print_graph_ops") - env = dict(os.environ) with as_file(files(__package__) / "ggml_ptn_runner") as runner: - # Shared libraries the runner needs may ship beside it. - env["LD_LIBRARY_PATH"] = ( - f"{Path(runner).parent}:{env.get('LD_LIBRARY_PATH', '')}" - ) result = subprocess.run( [str(runner), *command], capture_output=True, text=True, timeout=300, - env=env, ) if result.returncode != 0: raise AssertionError(f"ggml_ptn_runner failed:\n{result.stderr}") diff --git a/backends/native/test/runtime/vulkan/targets.bzl b/backends/native/test/runtime/vulkan/targets.bzl index 7ccf8b9c725..2a5cedef691 100644 --- a/backends/native/test/runtime/vulkan/targets.bzl +++ b/backends/native/test/runtime/vulkan/targets.bzl @@ -1,3 +1,4 @@ +load("@fbsource//tools/target_determinator/macros:ci.bzl", "ci") load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "is_xplat", "runtime") def define_common_targets(): @@ -38,6 +39,22 @@ def define_common_targets(): ], ) + # TODO: run in CI once it passes on SwiftShader. + runtime.cxx_test( + name = "vulkan_sample_test", + args = ["--packages=$(location //executorch/backends/native/test/samples:op_samples)"], + env = { + "ASAN_OPTIONS": "detect_leaks=0", + "LD_LIBRARY_PATH": "$(location :vulkan_test_driver)", + }, + labels = ci.labels(ci.map(ci.skip_test())), + test_main = "//executorch/backends/native/test/samples:sample_test_main", + deps = [ + ":vulkan_engine_host", + "fbsource//third-party/swiftshader/lib/linux-x64:libvk_swiftshader_fbcode", + ], + ) + runtime.cxx_test( name = "vulkan_engine_test", srcs = ["test_vulkan_engine.cpp"], diff --git a/backends/native/test/samples/BUCK b/backends/native/test/samples/BUCK index a59019b033c..122272f1d3e 100644 --- a/backends/native/test/samples/BUCK +++ b/backends/native/test/samples/BUCK @@ -66,3 +66,25 @@ fbcode_target( srcs = ["test_output_check.cpp"], deps = [":output_check"], ) + +# A gtest main judging the engine whose host library a test links on the +# bundled samples of the packages it names; engine tests set its flags. +fbcode_target( + _kind = runtime.cxx_library, + name = "sample_test_main", + srcs = ["sample_test_main.cpp"], + link_whole = True, + undefined_symbols = True, + visibility = ["//executorch/backends/native/test/..."], + exported_deps = ["//executorch/backends/native/test/runtime:engine_host"], + deps = [ + ":output_check", + "//executorch/backends/native/runtime:runtime", + "//executorch/backends/native/runtime/deserialize:package", + "//executorch/backends/native/runtime/deserialize:samples", + ], + external_deps = [ + "gflags", + "gtest", + ], +) diff --git a/backends/native/test/samples/sample_test_main.cpp b/backends/native/test/samples/sample_test_main.cpp new file mode 100644 index 00000000000..584676a49f6 --- /dev/null +++ b/backends/native/test/samples/sample_test_main.cpp @@ -0,0 +1,251 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +// A gtest main that runs the bundled samples of .ptn packages on the engine the +// binary links (see PtnRunner.h): one test per package and method, grouping +// samples by their first call. Each sample runs from a fresh context, and each +// method output is judged over all its samples by OutputCheck. +// +// Usage: +// --packages=a.ptn,dir [--rmse_ratio --max_ratio --rtol --sigmas] +// [engine host flags] [gtest flags] + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +DEFINE_string( + packages, + "", + "Comma-separated .ptn files or directories of them."); +DEFINE_double( + rmse_ratio, + ptn::testing::Policy{}.rmse_ratio, + "Allowed engine RMSE and biases, in multiples of eager's."); +DEFINE_double( + max_ratio, + ptn::testing::Policy{}.max_ratio, + "Allowed engine max abs error, in multiples of eager's."); +DEFINE_double( + rtol, + ptn::testing::Policy{}.rtol, + "Error allowed on top, relative to the precise outputs' magnitude."); +DEFINE_double( + sigmas, + ptn::testing::Policy{}.sigmas, + "Standard errors the fitted biases may deviate by."); + +namespace ptn::testing { +namespace { + +struct Bundle { + std::shared_ptr package; + std::shared_ptr program; + SampleSet golden; + SampleSet precise; +}; + +// Created on first use, so listing tests touches no device. +EngineHost& host() { + static const std::unique_ptr instance = [] { + std::unique_ptr made = make_engine_host(); + std::printf( + "engine %s on %s\n", made->name().c_str(), made->device_name().c_str()); + return made; + }(); + return *instance; +} + +// Keyed by method and output index. +using Checks = std::map, OutputCheck>; + +class SampleTest : public ::testing::Test { + public: + SampleTest(std::shared_ptr bundle, std::string method) + : bundle_(std::move(bundle)), method_(std::move(method)) {} + + void TestBody() override { + const std::vector& golden = bundle_->golden.samples; + const std::vector& precise = bundle_->precise.samples; + ASSERT_EQ(golden.size(), precise.size()); + Checks checks; + for (size_t s = 0; s < golden.size(); ++s) { + if (golden[s].calls.front().method != method_) { + continue; + } + SCOPED_TRACE("sample " + std::to_string(s)); + try { + run(golden[s], precise[s], checks); + } catch (const std::exception& e) { + FAIL() << e.what(); + } + if (HasFatalFailure()) { + return; + } + } + const Policy policy{ + .rmse_ratio = FLAGS_rmse_ratio, + .max_ratio = FLAGS_max_ratio, + .rtol = FLAGS_rtol, + .sigmas = FLAGS_sigmas}; + for (const auto& [key, check] : checks) { + const std::string name = key.first + " out " + std::to_string(key.second); + std::printf("%s: %s\n", name.c_str(), check.summary().c_str()); + for (const std::string& failure : check.failures(policy)) { + ADD_FAILURE() << name << ": " << failure; + } + } + } + + private: + void run(const Sample& golden, const Sample& precise, Checks& checks) { + ASSERT_EQ(golden.calls.size(), precise.calls.size()); + const std::unique_ptr context = + host().create_context(bundle_->program, bundle_->package); + std::map> executables; + for (size_t c = 0; c < golden.calls.size(); ++c) { + const SampleCall& call = golden.calls[c]; + ASSERT_EQ(call.method, precise.calls[c].method); + ASSERT_TRUE(call.inputs == precise.calls[c].inputs) + << call.method << " has golden and precise inputs that differ"; + ASSERT_TRUE(call.outputs && precise.calls[c].outputs) + << call.method << " has no recorded outputs"; + const std::vector& want = *call.outputs; + const std::vector& ref = *precise.calls[c].outputs; + std::unique_ptr& executable = executables[call.method]; + if (!executable) { + executable = context->compile(call.method); + } + ASSERT_EQ(executable->num_inputs(), call.inputs.size()); + ASSERT_EQ(executable->num_outputs(), want.size()); + ASSERT_EQ(ref.size(), want.size()); + + for (size_t i = 0; i < call.inputs.size(); ++i) { + const SampleTensor& in = call.inputs[i]; + ASSERT_STREQ( + scalar_type_name(executable->input_dtype(i)), + scalar_type_name(in.dtype)) + << "input " << i; + if (executable->input_sizes(i) != in.sizes) { + executable->resize_input(i, in.sizes); + } + executable->set_input( + i, + in.bytes.data(), + in.bytes.size() / element_size(in.dtype), + in.dtype); + } + executable->execute(); + for (size_t o = 0; o < want.size(); ++o) { + ASSERT_EQ(executable->output_sizes(o), want[o].sizes) << "output " << o; + ASSERT_EQ(ref[o].sizes, want[o].sizes) << "precise output " << o; + ASSERT_STREQ( + scalar_type_name(executable->output_dtype(o)), + scalar_type_name(want[o].dtype)) + << "output " << o; + std::vector got( + want[o].bytes.size() / element_size(want[o].dtype)); + executable->get_output(o, got.data(), got.size(), kFloat); + checks[{call.method, o}].add(got, want[o], ref[o]); + } + } + } + + std::shared_ptr bundle_; + std::string method_; +}; + +std::vector package_paths() { + std::vector paths; + std::stringstream list(FLAGS_packages); + for (std::string entry; std::getline(list, entry, ',');) { + if (entry.empty()) { + continue; + } + if (!std::filesystem::is_directory(entry)) { + paths.emplace_back(entry); + continue; + } + std::vector found; + for (const auto& file : std::filesystem::directory_iterator(entry)) { + if (file.path().extension() == ".ptn") { + found.push_back(file.path()); + } + } + std::ranges::sort(found); + paths.insert(paths.end(), found.begin(), found.end()); + } + return paths; +} + +void register_package(const std::filesystem::path& path) { + auto bundle = std::make_shared(); + bundle->package = + std::make_shared(Package::load(path.string())); + const ByteSpan program = bundle->package->program_bytes(); + bundle->program = std::make_shared( + Program::load(program.data(), program.size())); + const ZipReader zip = ZipReader::open(path.string()); + bundle->golden = read_sample_set(zip, "golden"); + bundle->precise = read_sample_set(zip, "precise"); + + std::set methods; + for (const Sample& sample : bundle->golden.samples) { + methods.insert(sample.calls.front().method); + } + const std::string suite = path.stem().string(); + for (const std::string& method : methods) { + ::testing::RegisterTest( + suite.c_str(), + method.c_str(), + nullptr, + nullptr, + __FILE__, + __LINE__, + [bundle, method]() { return new SampleTest(bundle, method); }); + } +} + +} // namespace +} // namespace ptn::testing + +int main(int argc, char** argv) { + ::testing::InitGoogleTest(&argc, argv); + gflags::ParseCommandLineFlags(&argc, &argv, true); + try { + const std::vector paths = + ptn::testing::package_paths(); + if (paths.empty()) { + throw std::invalid_argument("--packages names no .ptn"); + } + for (const std::filesystem::path& path : paths) { + ptn::testing::register_package(path); + } + } catch (const std::exception& e) { + std::fprintf(stderr, "error: %s\n", e.what()); + return 1; + } + return RUN_ALL_TESTS(); +}