[AUTOMATED] Add experimental C++ type graphs and scoped native provenance

.github/workflows/cpp-policy.yml

@@ -0,0 +1,75 @@
+name: C++ evaluation contracts
+on:
+  pull_request:
+  push:
+    branches: [main, 'codex/cpp-type-policy']
+  workflow_dispatch:
+permissions:
+  contents: read
+jobs:
+  contracts:
+    runs-on: ubuntu-24.04
+    timeout-minutes: 25
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+          cache: pip
+      - name: Native debug fixture compilers
+        run: sudo apt-get update && sudo apt-get install -y clang lld
+      - uses: dtolnay/rust-toolchain@stable
+      - name: Pinned existing CFG dependency
+        run: |
+          git clone https://github.com/Noelo-Lab/rust-joern.git ../rust-joern
+          git -C ../rust-joern checkout 9fd97f50885202596b125ebd2504e64910874bcb
+          cargo build --release --manifest-path ../rust-joern/Cargo.toml
+          python -m pip install ../rust-joern
+          test -f ../rust-joern/target/release/librust_joern.so
+      - name: Development dependencies
+        run: python -m pip install -e '.[dev]'
+      - name: Evaluation and provenance regressions
+        env:
+          DECBENCH_NO_CACHE: '1'
+          RUST_JOERN_LIBRARY: ${{ github.workspace }}/../rust-joern/target/release/librust_joern.so
+        run: |
+          python -m pytest --no-cov tests/test_cpp_first_pc_locations.py tests/test_cpp_native_inputs.py tests/test_cpp_toolchain_probe.py tests/test_producer_validation.py tests/test_cpp_type_graph.py tests/test_cpp_review_contracts.py \
+            tests/test_cpp_ged_identity.py tests/test_cpp_ged_compilation_units.py \
+            tests/test_recompiler_arch.py tests/test_byte_match_fixup_policy.py \
+            tests/test_metric_function_identity.py \
+            tests/test_cpp_support.py tests/test_cpp_identity_backends.py \
+            tests/test_cpp_identity_deep_validation.py tests/test_cpp_identity_end_to_end.py \
+            tests/test_cpp_identity_optimized_dwarf.py tests/test_cpp_identity_progress.py \
+            tests/test_cpp_identity_thumb.py tests/test_function_identity.py \
+            tests/test_cpp_identity_icf.py tests/test_cpp_identity_slice_invariance.py \
+            tests/test_type_match_correspondence.py tests/test_provenance_sanitizer.py \
+            tests/test_raw_line_provenance.py
+      - name: New policy code style
+        run: |
+          black --check tests/test_cpp_first_pc_locations.py
+          ruff check tests/test_cpp_first_pc_locations.py
+          black --check scripts/audit_cpp_toolchains.py tests/test_cpp_native_inputs.py tests/test_cpp_toolchain_probe.py decbench/models/decompilation.py decbench/decompilers/raw/angr_raw.py decbench/decompilers/raw/ghidra_raw.py decbench/decompilers/raw/kuna_raw.py decbench/utils/producer_validation.py tests/test_producer_validation.py decbench/utils/cfg_identity.py decbench/pipeline/cpp_ged.py \
+            scripts/evaluate_cpp_ged.py tests/test_cpp_ged_identity.py \
+            decbench/utils/compilation_units.py tests/test_cpp_ged_compilation_units.py \
+            decbench/metrics/variable_features.py \
+            decbench/utils/binfmt.py decbench/metrics/byte_match.py tests/test_recompiler_arch.py \
+            decbench/metrics/cpp_type_graph.py decbench/metrics/cpp_recovered_types.py \
+            decbench/metrics/cpp_types.py decbench/metrics/cpp_type_match.py \
+            decbench/utils/pdb_info.py decbench/decompilers/raw/fission_raw.py \
+            tests/test_cpp_type_graph.py tests/test_cpp_review_contracts.py
+          ruff check scripts/audit_cpp_toolchains.py tests/test_cpp_native_inputs.py tests/test_cpp_toolchain_probe.py decbench/models/decompilation.py decbench/decompilers/raw/angr_raw.py decbench/decompilers/raw/ghidra_raw.py decbench/decompilers/raw/kuna_raw.py decbench/utils/producer_validation.py tests/test_producer_validation.py decbench/utils/cfg_identity.py decbench/pipeline/cpp_ged.py \
+            scripts/evaluate_cpp_ged.py tests/test_cpp_ged_identity.py \
+            decbench/utils/compilation_units.py tests/test_cpp_ged_compilation_units.py \
+            decbench/metrics/variable_features.py \
+            decbench/utils/binfmt.py decbench/metrics/byte_match.py tests/test_recompiler_arch.py \
+            decbench/metrics/cpp_type_graph.py decbench/metrics/cpp_recovered_types.py \
+            decbench/metrics/cpp_types.py decbench/metrics/cpp_type_match.py \
+            decbench/utils/pdb_info.py decbench/decompilers/raw/fission_raw.py \
+            tests/test_cpp_type_graph.py tests/test_cpp_review_contracts.py
+      - name: New policy type contracts
+        run: |
+          mypy --follow-imports=silent scripts/audit_cpp_toolchains.py decbench/models/decompilation.py decbench/utils/producer_validation.py decbench/utils/cfg_identity.py \
+            decbench/utils/compilation_units.py \
+            decbench/pipeline/cpp_ged.py decbench/metrics/cpp_type_graph.py \
+            decbench/metrics/cpp_recovered_types.py decbench/metrics/cpp_types.py \
+            decbench/metrics/cpp_type_match.py

decbench/decompilers/external.py

@@ -82,12 +82,6 @@ def decompile_binary(
 
 
 EXTERNAL_DECOMPILERS: tuple[ExternalSpec, ...] = (
-    ExternalSpec(
-        id="fission",
-        display_name="Fission",
-        url="https://github.com/fission-systems/Fission",
-        version="0.2.3",
-    ),
     ExternalSpec(
         id="ventris",
         display_name="Ventris",

decbench/decompilers/provenance.py

@@ -115,6 +115,9 @@ def _hydrate_additive_provenance(
 ) -> None:
     """Hydrate additive fields absent from legacy pickled model instances."""
     for mapping in function.line_mappings:
+        if not hasattr(mapping, "defined_variables"):
+            mapping.defined_variables = None
+            report.increment("legacy_additive_fields_hydrated")
         if not isinstance(getattr(mapping, "addresses", None), list):
             mapping.addresses = []
             report.increment("legacy_additive_fields_hydrated")
@@ -179,6 +182,13 @@ def _sanitize_function(
     sanitized_mappings = []
     report.increment("line_mapping_rows_seen", len(function.line_mappings))
     for mapping in function.line_mappings:
+        if mapping.defined_variables is not None and not 1 <= mapping.line_number <= len(
+            function.decompiled_code.splitlines()
+        ):
+            modified = True
+            report.increment("line_mapping_rows_dropped")
+            report.record_address_drop("line_outside_output")
+            continue
         addresses = _filter_addresses(
             mapping.addresses,
             code=code,

decbench/decompilers/raw/__init__.py

@@ -19,6 +19,7 @@
     angr_raw,  # noqa: F401
     binja_raw,  # noqa: F401
     dewolf_raw,  # noqa: F401
+    fission_raw,  # noqa: F401
     ghidra_raw,  # noqa: F401
     glaurung_raw,  # noqa: F401
     ida_raw,  # noqa: F401
@@ -35,4 +36,5 @@
     "dewolf_raw",
     "manifold_raw",
     "glaurung_raw",
+    "fission_raw",
 ]

decbench/decompilers/raw/angr_raw.py

@@ -30,8 +30,10 @@
     DecompilerMetadata,
     FunctionDecompilation,
     LineMapping,
+    ProducerLanguageContract,
     VariableInfo,
 )
+from decbench.utils.function_identity import insert_function
 
 _l = logging.getLogger(__name__)
 
@@ -141,7 +143,7 @@ def _dump() -> None:
                     _l.debug("angr-raw: failed to decompile %s: %s", func_name, e)
 
                 if func_result is not None:
-                    decompiled_functions[func_name] = func_result
+                    insert_function(decompiled_functions, func_result)
                 else:
                     failed_functions.append(func_name)
                 _dump()
@@ -264,6 +266,9 @@ def _decompile_one(
             function_name=func_name,
         )
         metadata = common.extract_metrics(code)
+        metadata["output_contract"] = ProducerLanguageContract(
+            surface="c-like-pseudocode", declarations="partial", dialect="angr-c"
+        ).model_dump()
 
         return FunctionDecompilation(
             name=func_name,

decbench/decompilers/raw/binja_raw.py

@@ -35,6 +35,7 @@
     LineMapping,
     VariableInfo,
 )
+from decbench.utils.function_identity import insert_function
 
 _l = logging.getLogger(__name__)
 
@@ -149,8 +150,8 @@ def _dump() -> None:
             bv = self._load(binary_path)
             enumerated = self._enumerate(bv, elf_base, text_range, addr_targets)
             if functions is not None:
-                requested = {n for (n, _a) in functions}
-                enumerated = [(n, a) for (n, a) in enumerated if n in requested]
+                requested_addrs = {a for (_n, a) in functions}
+                enumerated = [(n, a) for (n, a) in enumerated if a in requested_addrs]
             enumerated = common.narrow_to_source(
                 enumerated,
                 function_names,
@@ -170,7 +171,7 @@ def _dump() -> None:
                     except Exception as e:  # noqa: BLE001
                         _l.debug("binja-raw: failed to decompile %s: %s", func_name, e)
                 if func_result is not None:
-                    decompiled_functions[func_name] = func_result
+                    insert_function(decompiled_functions, func_result)
                 else:
                     failed_functions.append(func_name)
                 _dump()

decbench/decompilers/raw/common.py

@@ -121,6 +121,12 @@ def elf_min_vaddr(binary_path: Path) -> int:
             if base is not None:
                 return base
             return 0
+        from decbench.utils import binfmt
+
+        info = binfmt.detect(binary_path)
+        if info is not None and info.fmt == "macho":
+            base = binfmt.mach_o_image_base(binary_path)
+            return base if base is not None else 0
         with open(binary_path, "rb") as f:
             elf = ELFFile(f)
             vaddrs = [seg["p_vaddr"] for seg in elf.iter_segments() if seg["p_type"] == "PT_LOAD"]

decbench/decompilers/raw/dewolf_raw.py

@@ -61,6 +61,7 @@
     FunctionDecompilation,
     VariableInfo,
 )
+from decbench.utils.function_identity import insert_function
 
 _l = logging.getLogger(__name__)
 
@@ -291,7 +292,7 @@ def _consume(addrs_arg: str) -> None:
                             metadata=common.extract_metrics(code),
                         )
                         with lock:
-                            decompiled_functions[name] = fd
+                            insert_function(decompiled_functions, fd)
                             _dump()
                     elif kind == "fail":
                         with lock: