diff --git a/backends/native/README.md b/backends/native/README.md index d3c9e87f283..c918183cddc 100644 --- a/backends/native/README.md +++ b/backends/native/README.md @@ -217,8 +217,9 @@ enabled, `native_runner_utils_test` covers their shared runner utilities. Run th with `ctest --test-dir cmake-out -R native_` after `cmake --build cmake-out`. The engines' sample tests judge them on the bundled op samples. Building them runs -the sample generator, which needs `torch` and `torchao`. Those with failing cases -are disabled in CTest; run their binaries directly with +the sample generator, which needs `torch`, `torchao` and the FACTO submodule at +`backends/cadence/utils/FACTO`. Those with failing cases are disabled in CTest; +run their binaries directly with `--packages=/backends/native/test/op_samples`. The `Test Native Engines` CI workflow has separate Linux jobs for the shared diff --git a/backends/native/test/CMakeLists.txt b/backends/native/test/CMakeLists.txt index 83e144150aa..523d450ba50 100644 --- a/backends/native/test/CMakeLists.txt +++ b/backends/native/test/CMakeLists.txt @@ -48,16 +48,18 @@ endforeach() # environment pins the eager kernels' ISA and threads, so the golden is the same # on any x86 host with AVX2. Other hosts can differ. set(_native_op_samples ${CMAKE_CURRENT_BINARY_DIR}/op_samples) -file(GLOB _native_op_samples_srcs CONFIGURE_DEPENDS +file(GLOB_RECURSE _native_op_samples_srcs CONFIGURE_DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/samples/*.py + ${EXECUTORCH_ROOT}/backends/cadence/utils/FACTO/facto/*.py ) add_custom_command( OUTPUT ${_native_op_samples}.stamp COMMAND ${CMAKE_COMMAND} -E env MKL_CBWR=COMPATIBLE ONEDNN_MAX_CPU_ISA=AVX2 ATEN_CPU_CAPABILITY=avx2 OMP_NUM_THREADS=1 MKL_NUM_THREADS=1 - "PYTHONPATH=${EXECUTORCH_ROOT}/src" ${PYTHON_EXECUTABLE} -m - executorch.backends.native.test.samples.generate --out ${_native_op_samples} + "PYTHONPATH=${EXECUTORCH_ROOT}/src:${EXECUTORCH_ROOT}/backends/cadence/utils/FACTO" + ${PYTHON_EXECUTABLE} -m executorch.backends.native.test.samples.generate + --out ${_native_op_samples} COMMAND ${CMAKE_COMMAND} -E touch ${_native_op_samples}.stamp DEPENDS ${_native_op_samples_srcs} ${EXECUTORCH_ROOT}/backends/test/suite/operators/test_linear.py diff --git a/backends/native/test/samples/BUCK b/backends/native/test/samples/BUCK index 122272f1d3e..326d33d26de 100644 --- a/backends/native/test/samples/BUCK +++ b/backends/native/test/samples/BUCK @@ -18,6 +18,7 @@ fbcode_target( "//executorch/backends/test/suite/operators:test_operators", "//executorch/exir/native:native", "//pytorch/ao:torchao", + "//pytorch/facto:facto", ], ) diff --git a/backends/native/test/samples/cases.py b/backends/native/test/samples/cases.py index b08baba963a..82b97316b24 100644 --- a/backends/native/test/samples/cases.py +++ b/backends/native/test/samples/cases.py @@ -7,11 +7,16 @@ """Op cases: a model and the inputs of each sample it is recorded with.""" import inspect +from collections.abc import Callable from dataclasses import dataclass from types import ModuleType +from typing import TYPE_CHECKING import torch +if TYPE_CHECKING: + from facto.inputgen.specs.model import Spec + Inputs = tuple[torch.Tensor, ...] # Caps a suite case's input and output elements over its samples, keeping at @@ -85,3 +90,20 @@ def suite_cases(module: ModuleType, samples: int) -> list[Case]: if not cases: raise ValueError(f"no test in {module.__name__} lowers a model") return cases + + +def facto_cases( + spec: "Spec", build: Callable[..., Case], keep: Callable[[int, tuple], bool] +) -> list[Case]: + """One-sample cases `build(name, *args)` from FACTO's argument tuples for + `spec`, named by position, keeping those where `keep(position, args)`.""" + # FACTO is on the path for the sample generator only, not for pytest. + from facto.inputgen.argtuple.gen import ArgumentTupleGenerator + from facto.inputgen.utils.random_manager import seeded_random_manager + + seeded_random_manager.seed(0) + return [ + build(f"facto_{i}", *args) + for i, (args, _, _) in enumerate(ArgumentTupleGenerator(spec).gen()) + if keep(i, args) + ] diff --git a/backends/native/test/samples/linear.py b/backends/native/test/samples/linear.py index 24c39fe2ccc..c1f1dc60e48 100644 --- a/backends/native/test/samples/linear.py +++ b/backends/native/test/samples/linear.py @@ -4,12 +4,76 @@ # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. -"""Linear cases: the backend test suite's, and shapes and inputs that stress -accumulation and quantization.""" +"""Linear cases: the backend test suite's, FACTO's, and shapes and inputs that +stress accumulation and quantization.""" import torch -from executorch.backends.native.test.samples.cases import Case, draws, suite_cases +from executorch.backends.native.test.samples.cases import ( + Case, + draws, + facto_cases, + suite_cases, +) from executorch.backends.test.suite.operators import test_linear +from facto.inputgen.argument.type import ArgType +from facto.inputgen.specs.model import ConstraintProducer as cp, InPosArg, OutArg, Spec + + +def _unit(*constraints) -> list: + """Float32 values in [-1, 1] under `constraints`.""" + return [ + cp.Dtype.Eq(lambda deps: torch.float32), + *constraints, + cp.Value.Ge(lambda deps, dtype, struct: -1), + cp.Value.Le(lambda deps, dtype, struct: 1), + ] + + +# in_features is a multiple of every scheme's group size, and out_features is +# large enough to fit an error's abs_bias and rel_bias from one sample. +_FACTO_SPEC = Spec( + op="linear.default", + inspec=[ + InPosArg( + ArgType.Tensor, + name="input", + deps=[1], + constraints=_unit( + cp.Rank.Ge(lambda deps: 1), + cp.Rank.Le(lambda deps: 3), + cp.Size.In( + lambda deps, r, d: [deps[0].size(1)] if d == r - 1 else range(1, 9) + ), + ), + ), + InPosArg( + ArgType.Tensor, + name="weight", + constraints=_unit( + cp.Rank.Eq(lambda deps: 2), + cp.Size.In(lambda deps, r, d: [128, 256, 384] if d else [16, 48, 96]), + ), + ), + InPosArg( + ArgType.TensorOpt, + name="bias", + deps=[1], + constraints=_unit( + cp.Rank.Eq(lambda deps: 1), + cp.Size.Eq(lambda deps, r, d: deps[0].size(0)), + ), + ), + ], + outspec=[OutArg(ArgType.Tensor)], +) + + +def _facto_case(name: str, x, weight, bias) -> Case: + model = torch.nn.Linear(weight.size(1), weight.size(0), bias=bias is not None) + model.weight.data = weight + if bias is not None: + model.bias.data = bias + return Case(name, model, [(x,)]) def cases() -> list[Case]: @@ -22,7 +86,8 @@ def cases() -> list[Case]: zero_row = draws(2, (8, 256)) for (x,) in zero_row: x[2] = 0 - return suite + [ + return [ + *suite, Case("decode", torch.nn.Linear(2048, 512), draws(8, (1, 2048))), Case("prefill", torch.nn.Linear(1024, 256), draws(2, (64, 1024))), Case("large_k", torch.nn.Linear(8192, 32), draws(4, (4, 8192))), @@ -33,4 +98,8 @@ def cases() -> list[Case]: ), Case("outlier_channel", torch.nn.Linear(256, 128), outlier), Case("zero_row", torch.nn.Linear(256, 64), zero_row), + # FACTO varies one argument at a time, so only every fifth tuple is kept. + # Its values lie on a 1/8 grid, which makes float32 math exact: these + # cases cover shapes more than rounding. + *facto_cases(_FACTO_SPEC, _facto_case, lambda i, _: i % 5 == 0), ]