[executorch][native] Add FACTO-drawn linear cases to the op samples

backends/native/README.md

@@ -217,8 +217,9 @@ enabled, `native_runner_utils_test` covers their shared runner utilities. Run th
 with `ctest --test-dir cmake-out -R native_` after `cmake --build cmake-out`.
 
 The engines' sample tests judge them on the bundled op samples. Building them runs
-the sample generator, which needs `torch` and `torchao`. Those with failing cases
-are disabled in CTest; run their binaries directly with
+the sample generator, which needs `torch`, `torchao` and the FACTO submodule at
+`backends/cadence/utils/FACTO`. Those with failing cases are disabled in CTest;
+run their binaries directly with
 `--packages=<build-dir>/backends/native/test/op_samples`.
 
 The `Test Native Engines` CI workflow has separate Linux jobs for the shared

backends/native/test/CMakeLists.txt

@@ -48,16 +48,18 @@ endforeach()
 # environment pins the eager kernels' ISA and threads, so the golden is the same
 # on any x86 host with AVX2. Other hosts can differ.
 set(_native_op_samples ${CMAKE_CURRENT_BINARY_DIR}/op_samples)
-file(GLOB _native_op_samples_srcs CONFIGURE_DEPENDS
+file(GLOB_RECURSE _native_op_samples_srcs CONFIGURE_DEPENDS
      ${CMAKE_CURRENT_SOURCE_DIR}/samples/*.py
+     ${EXECUTORCH_ROOT}/backends/cadence/utils/FACTO/facto/*.py
 )
 add_custom_command(
   OUTPUT ${_native_op_samples}.stamp
   COMMAND
     ${CMAKE_COMMAND} -E env MKL_CBWR=COMPATIBLE ONEDNN_MAX_CPU_ISA=AVX2
     ATEN_CPU_CAPABILITY=avx2 OMP_NUM_THREADS=1 MKL_NUM_THREADS=1
-    "PYTHONPATH=${EXECUTORCH_ROOT}/src" ${PYTHON_EXECUTABLE} -m
-    executorch.backends.native.test.samples.generate --out ${_native_op_samples}
+    "PYTHONPATH=${EXECUTORCH_ROOT}/src:${EXECUTORCH_ROOT}/backends/cadence/utils/FACTO"
+    ${PYTHON_EXECUTABLE} -m executorch.backends.native.test.samples.generate
+    --out ${_native_op_samples}
   COMMAND ${CMAKE_COMMAND} -E touch ${_native_op_samples}.stamp
   DEPENDS ${_native_op_samples_srcs}
           ${EXECUTORCH_ROOT}/backends/test/suite/operators/test_linear.py

backends/native/test/samples/BUCK

@@ -18,6 +18,7 @@ fbcode_target(
         "//executorch/backends/test/suite/operators:test_operators",
         "//executorch/exir/native:native",
         "//pytorch/ao:torchao",
+        "//pytorch/facto:facto",
     ],
 )
 

backends/native/test/samples/cases.py

@@ -7,11 +7,16 @@
 """Op cases: a model and the inputs of each sample it is recorded with."""
 
 import inspect
+from collections.abc import Callable
 from dataclasses import dataclass
 from types import ModuleType
+from typing import TYPE_CHECKING
 
 import torch
 
+if TYPE_CHECKING:
+    from facto.inputgen.specs.model import Spec
+
 Inputs = tuple[torch.Tensor, ...]
 
 # Caps a suite case's input and output elements over its samples, keeping at
@@ -85,3 +90,20 @@ def suite_cases(module: ModuleType, samples: int) -> list[Case]:
     if not cases:
         raise ValueError(f"no test in {module.__name__} lowers a model")
     return cases
+
+
+def facto_cases(
+    spec: "Spec", build: Callable[..., Case], keep: Callable[[int, tuple], bool]
+) -> list[Case]:
+    """One-sample cases `build(name, *args)` from FACTO's argument tuples for
+    `spec`, named by position, keeping those where `keep(position, args)`."""
+    # FACTO is on the path for the sample generator only, not for pytest.
+    from facto.inputgen.argtuple.gen import ArgumentTupleGenerator
+    from facto.inputgen.utils.random_manager import seeded_random_manager
+
+    seeded_random_manager.seed(0)
+    return [
+        build(f"facto_{i}", *args)
+        for i, (args, _, _) in enumerate(ArgumentTupleGenerator(spec).gen())
+        if keep(i, args)
+    ]

backends/native/test/samples/linear.py

@@ -4,12 +4,76 @@
 # This source code is licensed under the BSD-style license found in the
 # LICENSE file in the root directory of this source tree.
 
-"""Linear cases: the backend test suite's, and shapes and inputs that stress
-accumulation and quantization."""
+"""Linear cases: the backend test suite's, FACTO's, and shapes and inputs that
+stress accumulation and quantization."""
 
 import torch
-from executorch.backends.native.test.samples.cases import Case, draws, suite_cases
+from executorch.backends.native.test.samples.cases import (
+    Case,
+    draws,
+    facto_cases,
+    suite_cases,
+)
 from executorch.backends.test.suite.operators import test_linear
+from facto.inputgen.argument.type import ArgType
+from facto.inputgen.specs.model import ConstraintProducer as cp, InPosArg, OutArg, Spec
+
+
+def _unit(*constraints) -> list:
+    """Float32 values in [-1, 1] under `constraints`."""
+    return [
+        cp.Dtype.Eq(lambda deps: torch.float32),
+        *constraints,
+        cp.Value.Ge(lambda deps, dtype, struct: -1),
+        cp.Value.Le(lambda deps, dtype, struct: 1),
+    ]
+
+
+# in_features is a multiple of every scheme's group size, and out_features is
+# large enough to fit an error's abs_bias and rel_bias from one sample.
+_FACTO_SPEC = Spec(
+    op="linear.default",
+    inspec=[
+        InPosArg(
+            ArgType.Tensor,
+            name="input",
+            deps=[1],
+            constraints=_unit(
+                cp.Rank.Ge(lambda deps: 1),
+                cp.Rank.Le(lambda deps: 3),
+                cp.Size.In(
+                    lambda deps, r, d: [deps[0].size(1)] if d == r - 1 else range(1, 9)
+                ),
+            ),
+        ),
+        InPosArg(
+            ArgType.Tensor,
+            name="weight",
+            constraints=_unit(
+                cp.Rank.Eq(lambda deps: 2),
+                cp.Size.In(lambda deps, r, d: [128, 256, 384] if d else [16, 48, 96]),
+            ),
+        ),
+        InPosArg(
+            ArgType.TensorOpt,
+            name="bias",
+            deps=[1],
+            constraints=_unit(
+                cp.Rank.Eq(lambda deps: 1),
+                cp.Size.Eq(lambda deps, r, d: deps[0].size(0)),
+            ),
+        ),
+    ],
+    outspec=[OutArg(ArgType.Tensor)],
+)
+
+
+def _facto_case(name: str, x, weight, bias) -> Case:
+    model = torch.nn.Linear(weight.size(1), weight.size(0), bias=bias is not None)
+    model.weight.data = weight
+    if bias is not None:
+        model.bias.data = bias
+    return Case(name, model, [(x,)])
 
 
 def cases() -> list[Case]:
@@ -22,7 +86,8 @@ def cases() -> list[Case]:
     zero_row = draws(2, (8, 256))
     for (x,) in zero_row:
         x[2] = 0
-    return suite + [
+    return [
+        *suite,
         Case("decode", torch.nn.Linear(2048, 512), draws(8, (1, 2048))),
         Case("prefill", torch.nn.Linear(1024, 256), draws(2, (64, 1024))),
         Case("large_k", torch.nn.Linear(8192, 32), draws(4, (4, 8192))),
@@ -33,4 +98,8 @@ def cases() -> list[Case]:
         ),
         Case("outlier_channel", torch.nn.Linear(256, 128), outlier),
         Case("zero_row", torch.nn.Linear(256, 64), zero_row),
+        # FACTO varies one argument at a time, so only every fifth tuple is kept.
+        # Its values lie on a 1/8 grid, which makes float32 math exact: these
+        # cases cover shapes more than rounding.
+        *facto_cases(_FACTO_SPEC, _facto_case, lambda i, _: i % 5 == 0),
     ]