FFL-3568: Define shared mobile precomputed response conformance

.github/workflows/mobile-compatibility.yml

@@ -0,0 +1,136 @@
+name: Mobile compatibility preview
+
+on:
+  pull_request:
+    paths:
+      - .github/workflows/mobile-compatibility.yml
+      - ci/mobile_conformance.py
+      - ci/test_mobile_conformance.py
+      - ci/sync-precomputed-fixtures.py
+      - ci/precomputed_obfuscation.py
+      - schemas/precomputed-assignment.schema.json
+      - precomputed-assignments/**
+
+permissions:
+  contents: read
+
+concurrency:
+  group: mobile-compatibility-${{ github.event.pull_request.number }}
+  cancel-in-progress: true
+
+jobs:
+  tooling:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+      - run: python3 -m unittest discover -s ci -p 'test_mobile_conformance.py'
+
+  mobile:
+    name: ${{ matrix.platform }} unit compatibility (advisory)
+    runs-on: ${{ matrix.runner }}
+    timeout-minutes: 90
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          # Temporary development branches. Switch each ref to develop after the
+          # canonical runner lands; the checked-out SDK SHA stays fixed per run.
+          - platform: ios
+            runner: macos-26
+            repository: DataDog/dd-sdk-ios
+            ref: leo.romanovsky/precompute-fixture-fanout
+          - platform: android
+            runner: ubuntu-latest
+            repository: DataDog/dd-sdk-android
+            ref: leo.romanovsky/ffl-3568/precompute-fixture-fanout
+    steps:
+      - name: Check out fixture PR
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          fetch-depth: 0
+          path: fixtures
+          persist-credentials: false
+      - name: Check out SDK once for both fixture revisions
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          repository: ${{ matrix.repository }}
+          ref: ${{ matrix.ref }}
+          path: downstream
+          persist-credentials: false
+      - name: Record revisions
+        id: revisions
+        env:
+          BASE_SHA: ${{ github.event.pull_request.base.sha }}
+          HEAD_SHA: ${{ github.event.pull_request.head.sha }}
+          SDK_REPOSITORY: ${{ matrix.repository }}
+          SDK_REF: ${{ matrix.ref }}
+        run: |
+          mkdir -p "$RUNNER_TEMP/mobile-conformance"
+          base=$(git -C fixtures merge-base "$BASE_SHA" "$HEAD_SHA")
+          echo "base=$base" >> "$GITHUB_OUTPUT"
+          {
+            echo "SDK: $SDK_REPOSITORY @ $SDK_REF"
+            git -C downstream rev-parse HEAD
+            echo "Base fixtures: $base"
+            echo "Proposed fixtures: $HEAD_SHA"
+          } | tee "$RUNNER_TEMP/mobile-conformance/revisions.txt" >> "$GITHUB_STEP_SUMMARY"
+      - name: Set up Java
+        if: matrix.platform == 'android'
+        uses: actions/setup-java@de7274f081f381c8f8158605e0321c36c376e2e6 # v6.0.1
+        with:
+          distribution: temurin
+          java-version: '21'
+      - name: Prepare Android build tools
+        if: matrix.platform == 'android'
+        run: |
+          set -o pipefail
+          {
+            java -version
+            "$ANDROID_HOME/cmdline-tools/latest/bin/sdkmanager" \
+              'platforms;android-36' 'build-tools;36.0.0'
+          } 2>&1 | tee "$RUNNER_TEMP/mobile-conformance/setup.log"
+      - name: Set up Ruby for iOS repository setup
+        if: matrix.platform == 'ios'
+        uses: ruby/setup-ruby@a0102e0972be65f351c307e2d64b9314a57c8073 # v1.324.0
+        with:
+          ruby-version: '3.3'
+      - name: Prepare iOS dependencies
+        if: matrix.platform == 'ios'
+        working-directory: downstream
+        env:
+          DEVELOPER_DIR: /Applications/Xcode_26.6.app/Contents/Developer
+        run: |
+          set -o pipefail
+          {
+            xcodebuild -version
+            xcrun simctl list devices available
+            make repo-setup ENV=ci
+            # Public dependencies; the SDK's GitLab STS wrapper is not needed here.
+            carthage bootstrap --platform iOS --use-xcframeworks
+          } 2>&1 | tee "$RUNNER_TEMP/mobile-conformance/setup.log"
+      - name: Compare native unit-test results
+        id: comparison
+        env:
+          PLATFORM: ${{ matrix.platform }}
+          BASE_SHA: ${{ steps.revisions.outputs.base }}
+          HEAD_SHA: ${{ github.event.pull_request.head.sha }}
+          DEVELOPER_DIR: /Applications/Xcode_26.6.app/Contents/Developer
+          IOS_DESTINATION: platform=iOS Simulator,name=iPhone 17 Pro,OS=26.5
+        run: |
+          python3 fixtures/ci/mobile_conformance.py \
+            --platform "$PLATFORM" --fixtures "$GITHUB_WORKSPACE/fixtures" \
+            --sdk "$GITHUB_WORKSPACE/downstream" --base "$BASE_SHA" --head "$HEAD_SHA" \
+            --output "$RUNNER_TEMP/mobile-conformance"
+      - name: Report unavailable preview
+        if: failure() && steps.comparison.outcome != 'success'
+        run: |
+          echo 'Mobile preview incomplete: inspect setup/build logs. This is not evidence of fixture compatibility.' >> "$GITHUB_STEP_SUMMARY"
+      - name: Upload native results and logs
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: mobile-compatibility-${{ matrix.platform }}
+          path: ${{ runner.temp }}/mobile-conformance
+          if-no-files-found: warn

.github/workflows/validate-precomputed-fixtures.yml

@@ -5,6 +5,9 @@ on:
     paths:
       - .github/workflows/validate-precomputed-fixtures.yml
       - ci/validate-precomputed-fixtures.py
+      - ci/test_precomputed*.py
+      - ci/precomputed_obfuscation.py
+      - ci/sync-precomputed-fixtures.py
       - schemas/precomputed-assignment.schema.json
       - precomputed-assignments/**
 
@@ -32,3 +35,9 @@ jobs:
 
       - name: Validate precomputed fixtures
         run: python3 ci/validate-precomputed-fixtures.py
+
+      - name: Verify plain and obfuscated fixture parity
+        run: python3 ci/precomputed_obfuscation.py --check
+
+      - name: Test fixture validation and snapshot import
+        run: python3 -m unittest discover -s ci -p 'test_precomputed*.py'

AGENTS.md

@@ -140,14 +140,14 @@ Always `[{"platform": "<id>", "reason": "<optional>"}]`. Mutually exclusive.
 Default (neither present): applies to all platforms.
 
 Valid platforms: `web`, `ios`, `android`, `react-native`, `flutter`, `maui`,
-`kotlin-multiplatform`, `electron`.
+`kotlin-multiplatform`, `electron`, `unity`.
 
 ## Error codes
 
 | Code | When |
 |------|------|
 | `FLAG_NOT_FOUND` | Flag key not in response, or flag dropped due to malformed data |
-| `TYPE_MISMATCH` | Requested type does not match `variationType` |
+| `TYPE_MISMATCH` | Incompatible getter or numeric conversion loses information |
 | `PARSE_ERROR` | Enumerated field contains unexpected value |
 | `TARGETING_KEY_MISSING` | Evaluation requires targeting key but none provided |
 
@@ -207,3 +207,18 @@ Example — web aggregates 2 evaluations into 1 event:
   {"_count": 1, "_include": [{"platform": "web"}], "evaluation_count": 2}
 ]
 ```
+
+## Obfuscated companions
+
+Generate a companion for each plain precomputed fixture:
+
+```bash
+python3 ci/precomputed_obfuscation.py --write
+python3 ci/precomputed_obfuscation.py --check
+```
+
+Do not edit `precomputed-assignments/cases/obfuscation/generated/` manually.
+Keep rejection fixtures and fixed digest vectors outside that generated folder.
+CI checks companion content, not just file existence. A new case is discovered
+without a filename list. Read [the coverage policy](precomputed-assignments/README.md#plain-and-obfuscated-coverage)
+for the explicit exception format and platform applicability rules.

ci/mobile_conformance.py

@@ -0,0 +1,179 @@
+#!/usr/bin/env python3
+# Unless explicitly stated otherwise, all files in this repository are licensed under the
+# Apache License Version 2.0. This product includes software developed at Datadog
+# (https://www.datadoghq.com/). Copyright 2025-Present Datadog, Inc.
+"""Preview base/head precomputed fixtures against one disposable native SDK checkout."""
+import argparse
+import importlib.util
+import json
+import os
+from pathlib import Path
+import shutil
+import subprocess
+import xml.etree.ElementTree as ET
+
+spec = importlib.util.spec_from_file_location(
+    'snapshot_importer', Path(__file__).with_name('sync-precomputed-fixtures.py'))
+importer = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(importer)
+
+DESTINATIONS = {
+    'ios': 'DatadogFlags/Tests/Fixtures/precomputed',
+    'android': 'reliability/single-fit/flags/src/test/resources/precomputed',
+}
+ANDROID_MODULE = 'reliability/single-fit/flags'
+ANDROID_TEST = 'com.datadog.android.flags.integration.CanonicalPrecomputedTest'
+
+
+def install_snapshot(fixtures, sha, sdk, platform):
+    # Reuse the byte-preserving importer, including for bases predating its CLI.
+    # Never accept a configurable replacement destination in a downstream repo.
+    content = importer.snapshot(fixtures, sha)
+    destination = sdk / DESTINATIONS[platform]
+    if destination.resolve() != destination or any(p.is_symlink() for p in destination.parents):
+        raise ValueError('Snapshot destination must not contain symlinks')
+    if destination.exists():
+        if not (destination / 'SOURCE.json').is_file():
+            raise ValueError('Refusing to replace a directory without snapshot provenance')
+        shutil.rmtree(destination)
+    destination.mkdir(parents=True)
+    for name, data in content.items():
+        path = destination / name
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_bytes(data)
+    return len(json.loads(content['index.json']))
+
+
+def native_command(platform, output):
+    if platform == 'android':
+        return ['./gradlew', ':reliability:single-fit:flags:cleanTestDebugUnitTest',
+                ':reliability:single-fit:flags:testDebugUnitTest', '--tests', ANDROID_TEST,
+                '--no-daemon', '--no-build-cache', '--console=plain']
+    return ['xcodebuild', '-workspace', 'Datadog.xcworkspace', '-scheme', 'DatadogFlags',
+            '-destination', os.environ['IOS_DESTINATION'],
+            '-only-testing:DatadogFlagsTests/CanonicalPrecomputedTests',
+            '-resultBundlePath', str(output / 'tests.xcresult'),
+            'CODE_SIGNING_ALLOWED=NO', 'test']
+
+
+def android_results(directory):
+    cases = []
+    for file in directory.glob('TEST-*.xml'):
+        cases.extend(ET.parse(file).getroot().iter('testcase'))
+    # Require actual canonical cases, not an unrelated successful test report.
+    cases = [c for c in cases if c.get('classname') == ANDROID_TEST]
+    executed = [c for c in cases if c.find('skipped') is None]
+    failures = [c for c in executed if c.find('failure') is not None or c.find('error') is not None]
+    return {'executed': len(executed), 'failed': len(failures),
+            'failures': [c.get('name') for c in failures]}
+
+
+def ios_results(output):
+    raw = subprocess.check_output(['xcrun', 'xcresulttool', 'get', 'test-results', 'summary',
+                                   '--path', str(output / 'tests.xcresult')])
+    (output / 'test-summary.json').write_bytes(raw)
+    result = json.loads(raw)
+    return {'executed': result['passedTests'] + result['failedTests'],
+            'failed': result['failedTests'], 'failures': result.get('testFailures', [])}
+
+
+def test_status(exit_code, report):
+    if report['executed'] == 0:
+        return 'no-tests' if exit_code == 0 else 'build-or-runner-failure'
+    if report['failed']:
+        return 'test-failure'
+    return 'pass' if exit_code == 0 else 'build-or-runner-failure'
+
+
+def compare(base, head):
+    if base == head == 'pass':
+        return 'compatible'
+    if base == 'pass' and head == 'test-failure':
+        return 'new-regression'
+    if base == 'test-failure' and head == 'pass':
+        return 'improvement'
+    if base == head == 'test-failure':
+        return 'inconclusive-both-fail'
+    return 'inconclusive-setup-or-execution'
+
+
+def run_revision(args, label, sha):
+    output = args.output / label
+    output.mkdir()  # Never reuse stale results from an earlier attempt.
+    result = {'fixture_sha': sha, 'status': 'setup-failure'}
+    try:
+        result['fixture_count'] = install_snapshot(args.fixtures, sha, args.sdk, args.platform)
+        command = native_command(args.platform, output)
+        result['command'] = command
+        if args.platform == 'android':
+            # A compilation failure must not reuse the base run's JUnit report.
+            reports = args.sdk / ANDROID_MODULE / 'build/test-results/testDebugUnitTest'
+            if reports.exists():
+                shutil.rmtree(reports)
+        env = {**os.environ, 'SKIP_LINT': '1', 'USE_TEST_VISIBILITY': '0'}
+        with (output / 'test.log').open('w') as log:
+            result['exit_code'] = subprocess.run(command, cwd=args.sdk, env=env,
+                                                  stdout=log, stderr=subprocess.STDOUT).returncode
+        result['status'] = 'build-or-runner-failure'
+        if args.platform == 'android':
+            reports = args.sdk / ANDROID_MODULE / 'build/test-results/testDebugUnitTest'
+            if reports.exists():
+                shutil.copytree(reports, output / 'junit')
+            report = android_results(output / 'junit')
+        else:
+            report = ios_results(output)
+        result.update(report)
+        result['status'] = test_status(result['exit_code'], report)
+    except (OSError, ValueError, KeyError, ET.ParseError, subprocess.CalledProcessError) as error:
+        result['error'] = str(error)
+    (output / 'result.json').write_text(json.dumps(result, indent=2) + '\n')
+    print(f'{label}: {result["status"]}', flush=True)
+    return result
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('--platform', choices=DESTINATIONS, required=True)
+    for name in ('fixtures', 'sdk', 'output'):
+        parser.add_argument('--' + name, type=Path, required=True)
+    parser.add_argument('--base', required=True)
+    parser.add_argument('--head', required=True)
+    args = parser.parse_args()
+    args.fixtures, args.sdk, args.output = (p.resolve() for p in (args.fixtures, args.sdk, args.output))
+    args.output.mkdir(parents=True, exist_ok=True)
+    sdk_sha = importer.git(args.sdk, 'rev-parse', 'HEAD').decode().strip()
+    # Resolve immutable revisions up front. The SDK checkout never changes between runs.
+    base_sha, head_sha = [importer.git(args.fixtures, 'rev-parse', '--verify', s + '^{commit}')
+                          .decode().strip() for s in (args.base, args.head)]
+    base = run_revision(args, 'base', base_sha)
+    head = run_revision(args, 'head', head_sha)
+    classification = compare(base['status'], head['status'])
+    result = {'platform': args.platform, 'sdk_sha': sdk_sha,
+              'classification': classification, 'base': base, 'head': head}
+    (args.output / 'comparison.json').write_text(json.dumps(result, indent=2) + '\n')
+    summary = (f'### {args.platform} unit compatibility: {classification}\n\n'
+               f'SDK commit: `{sdk_sha}`\n\n'
+               '| Fixtures | Revision | Result | Executed tests |\n'
+               '| --- | --- | --- | ---: |\n'
+               f'| Base | `{base_sha}` | {base["status"]} | {base.get("executed", "unknown")} |\n'
+               f'| Proposed | `{head_sha}` | {head["status"]} | {head.get("executed", "unknown")} |\n\n'
+               'Advisory unit-test evidence, not deployed-client validation. '
+               'Both failing does not establish that the failures are identical. '
+               'Inspect the per-revision native reports and logs.\n')
+    for label, revision in [('Base', base), ('Proposed', head)]:
+        if revision.get('failures'):
+            # JSON escaping prevents test names from injecting Markdown/HTML.
+            details = json.dumps(revision['failures'][:20], indent=2).replace('`', '\\u0060').replace('<', '\\u003c')
+            summary += f'\n{label} failures (first 20):\n\n```json\n{details}\n```\n'
+    print(summary)
+    if os.environ.get('GITHUB_STEP_SUMMARY'):
+        with open(os.environ['GITHUB_STEP_SUMMARY'], 'a') as file:
+            file.write(summary)
+    if classification != 'compatible':
+        print(f'::warning title=Mobile fixture preview::{args.platform}: {classification}; see artifacts.')
+    # Completed comparisons are advisory. Setup/execution gaps must stay visibly failed.
+    return 1 if classification == 'inconclusive-setup-or-execution' else 0
+
+
+if __name__ == '__main__':
+    raise SystemExit(main())